From 5d1b1ab5547726185114147152f51d71b63c5c0f Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Thu, 15 Feb 2024 13:38:37 +0100 Subject: [PATCH 01/16] Add "app upload" + "mysql import" commands --- src/commands/app/upload.tsx | 100 +++++++++++++++ src/commands/database/mysql/import.tsx | 169 +++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 src/commands/app/upload.tsx create mode 100644 src/commands/database/mysql/import.tsx diff --git a/src/commands/app/upload.tsx b/src/commands/app/upload.tsx new file mode 100644 index 00000000..f1ee44e9 --- /dev/null +++ b/src/commands/app/upload.tsx @@ -0,0 +1,100 @@ +import { ExecRenderBaseCommand } from "../../rendering/react/ExecRenderBaseCommand.js"; +import { appInstallationArgs } from "../../lib/app/flags.js"; +import { + makeProcessRenderer, + processFlags, +} from "../../rendering/process/process_flags.js"; +import { Flags } from "@oclif/core"; +import { Success } from "../../rendering/react/components/Success.js"; +import { ReactNode } from "react"; +import { hasBinary } from "../../lib/hasbin.js"; +import { getSSHConnectionForAppInstallation } from "../../lib/ssh/appinstall.js"; +import { spawnInProcess } from "../../rendering/process/process_exec.js"; + +export class Upload extends ExecRenderBaseCommand { + static summary = "Upload the filesystem of an app to a project"; + static description = + "Upload the filesystem of an app from your local machine to a project." + + "" + + "CAUTION: This is a potentially destructive operation. It will overwrite files on the server with the files from your local machine." + + "This is NOT a turnkey deployment solution. It is intended for development purposes only."; + static args = { + ...appInstallationArgs, + }; + static flags = { + ...processFlags, + "dry-run": Flags.boolean({ + description: "do not actually download the app installation", + default: false, + }), + delete: Flags.boolean({ + description: "delete local files that are not present on the server", + default: false, + }), + source: Flags.directory({ + description: "source directory from which to upload the app installation", + required: true, + exists: false, + }), + }; + + protected async exec(): Promise { + const appInstallationId = await this.withAppInstallationId(Upload); + const { "dry-run": dryRun, source, delete: deleteRemote } = this.flags; + + const p = makeProcessRenderer(this.flags, "Uploading app installation"); + + const { host, user, directory } = await p.runStep( + "getting connection data", + async () => { + return getSSHConnectionForAppInstallation( + this.apiClient, + appInstallationId, + ); + }, + ); + + await p.runStep("check if rsync is installed", async () => { + if (!(await hasBinary("rsync"))) { + throw new Error("this command requires rsync to be installed"); + } + }); + + const rsyncOpts = [ + "--archive", + "--recursive", + "--verbose", + "--progress", + "--exclude=typo3temp", + ]; + if (dryRun) { + rsyncOpts.push("--dry-run"); + } + if (deleteRemote) { + rsyncOpts.push("--delete"); + } + + await spawnInProcess( + p, + "uploading app installation" + (dryRun ? " (dry-run)" : ""), + "rsync", + [...rsyncOpts, source, `${user}@${host}:${directory}/`], + ); + + if (dryRun) { + await p.complete( + + App would (probably) have successfully been uploaded. 🙂 + , + ); + } else { + await p.complete( + App successfully uploaded; have fun! 🚀, + ); + } + } + + protected render(): ReactNode { + return undefined; + } +} diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx new file mode 100644 index 00000000..8e0b5e42 --- /dev/null +++ b/src/commands/database/mysql/import.tsx @@ -0,0 +1,169 @@ +import { ExecRenderBaseCommand } from "../../../rendering/react/ExecRenderBaseCommand.js"; +import { Flags } from "@oclif/core"; +import { ReactNode } from "react"; +import { + makeProcessRenderer, + processFlags, +} from "../../../rendering/process/process_flags.js"; +import { Text } from "ink"; +import { Value } from "../../../rendering/react/components/Value.js"; +import * as fs from "fs"; +import { Success } from "../../../rendering/react/components/Success.js"; +import { + mysqlArgs, + mysqlConnectionFlags, + withMySQLId, +} from "../../../lib/database/mysql/flags.js"; +import { getConnectionDetailsWithPassword } from "../../../lib/database/mysql/connect.js"; +import { assertStatus } from "@mittwald/api-client"; +import { randomBytes } from "crypto"; +import { executeViaSSH } from "../../../lib/ssh/exec.js"; +import assertSuccess from "../../../lib/assert_success.js"; + +export class Import extends ExecRenderBaseCommand< + typeof Import, + Record +> { + static summary = "Imports a dump of a MySQL database"; + static flags = { + ...processFlags, + ...mysqlConnectionFlags, + "temporary-user": Flags.boolean({ + summary: "create a temporary user for the dump", + description: + "Create a temporary user for the dump. This user will be deleted after the dump has been imported. This is useful if you want to import a dump into a database that is not accessible from the outside.\n\nIf this flag is disabled, you will need to specify the password of the default user; either via the --mysql-password flag or via the MYSQL_PWD environment variable.", + default: true, + required: false, + allowNo: true, + }), + input: Flags.string({ + char: "i", + summary: 'the input file from which to read the dump ("-" for stdin)', + description: + 'The input file from which to read the dump to. You can specify "-" or "/dev/stdin" to read the dump directly from STDIN.', + required: true, + }), + }; + static args = { ...mysqlArgs }; + + protected async exec(): Promise> { + const databaseId = await withMySQLId( + this.apiClient, + this.flags, + this.args, + this.config, + ); + const p = makeProcessRenderer(this.flags, "Importing a MySQL database"); + + const connectionDetails = await getConnectionDetailsWithPassword( + this.apiClient, + databaseId, + p, + this.flags, + ); + + if (this.flags["temporary-user"]) { + const [tempUser, tempPassword] = await p.runStep( + "creating a temporary database user", + () => this.createTemporaryUser(databaseId), + ); + + p.addCleanup("removing temporary database user", async () => { + const r = await this.apiClient.database.deleteMysqlUser({ + mysqlUserId: tempUser.id, + }); + assertSuccess(r); + }); + + connectionDetails.user = tempUser.name; + connectionDetails.password = tempPassword; + } + + const { project } = connectionDetails; + const mysqlArgs = buildMySqlArgs(connectionDetails); + + await p.runStep( + + starting mysql via SSH on project {project.shortId} + , + () => + executeViaSSH( + this.apiClient, + { projectId: connectionDetails.project.id }, + { command: "mysql", args: mysqlArgs }, + null, + this.getInputStream(), + ), + ); + + await p.complete( + , + ); + + return {}; + } + + protected render(): ReactNode { + return undefined; + } + + private getInputStream(): NodeJS.ReadableStream { + if (this.flags.input === "-") { + return process.stdin; + } + + return fs.createReadStream(this.flags.input); + } + + private async createTemporaryUser( + databaseId: string, + ): Promise<[{ id: string; name: string }, string]> { + const password = randomBytes(32).toString("base64"); + const createResponse = await this.apiClient.database.createMysqlUser({ + mysqlDatabaseId: databaseId, + data: { + accessLevel: "full", // needed for "PROCESS" privilege + externalAccess: false, + password, + databaseId, + description: "Temporary user for exporting database", + }, + }); + + assertStatus(createResponse, 201); + + const userResponse = await this.apiClient.database.getMysqlUser({ + mysqlUserId: createResponse.data.id, + }); + assertStatus(userResponse, 200); + + return [userResponse.data, password]; + } +} + +function ImportSuccess({ + database, + input, +}: { + database: string; + input: string; +}) { + return ( + + Dump of MySQL database {database} successfully imported{" "} + from {input} + + ); +} + +function buildMySqlArgs(d: { + hostname: string; + user: string; + password: string; + database: string; +}): string[] { + return ["-h", d.hostname, "-u", d.user, "-p" + d.password, d.database]; +} From b952e469040ce9e5cebb283e03078f6a8c8813ff Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Thu, 15 Feb 2024 13:28:33 +0100 Subject: [PATCH 02/16] Add option to run "database mysql dump" command with --gzip flag --- ...escape-npm-0.2.3-4d809b6aa3-05065630b5.zip | Bin 0 -> 3069 bytes ...escape-npm-0.2.0-38c5261c29-0d87f1ae22.zip | Bin 0 -> 2994 bytes package.json | 2 ++ src/commands/database/mysql/dump.tsx | 24 +++++++++++++++--- src/lib/ssh/exec.ts | 13 +++++++--- yarn.lock | 16 ++++++++++++ 6 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 .yarn/cache/@types-shell-escape-npm-0.2.3-4d809b6aa3-05065630b5.zip create mode 100644 .yarn/cache/shell-escape-npm-0.2.0-38c5261c29-0d87f1ae22.zip diff --git a/.yarn/cache/@types-shell-escape-npm-0.2.3-4d809b6aa3-05065630b5.zip b/.yarn/cache/@types-shell-escape-npm-0.2.3-4d809b6aa3-05065630b5.zip new file mode 100644 index 0000000000000000000000000000000000000000..a05d4f7ef4203400d99e3c118533012d00fcc49b GIT binary patch literal 3069 zcma);2{hE}8^;GT4HLqDFf~I|_C}_Wt-98+%ov8jC58;L4KtQ3qsZEZN=^Bz+Ys9uuOPdg^pU-iBXNMzy1nout4 zF^>fbbYdw*x2QE}{$h_D!_L^r6*wGl1G)<`kSv#!C&Q8qObn!&>PAhz17Qb(MF&yj zVTUy8&6CeB1wT;vO3|&|#s)-wn3&_288+!$*omk@jl@6WyQi7V%Zv=5pS}2jgX5im z?cu&@AkzO<61R^AzafTR)&mDFJc}987}z7TkpC#74SJPFt-li(6dxIm@iW+OpFp7Y zqfq@kQ>#9_s&H^%+z9^M%YesMM+vI-A@lQU;K*)aL;d$f%r2AWT!(foeDFfH)LJ$ta7>#>2||v z9?$v=5nxmv=i(1-KerM97 zrN|2BP3=b-49Li>TS50FerAhYm7RxFu$Hiu-pUl`!ZEg;uYc2ZQ0BPO zaeF1R6ZgqdA>%w!SKQl;ZWa;QUTP&qiKB_0k_s2DJ;f^QM;o&iz^7z(WpEIgS>r6m z{>l5rWW8p+DE5U8~#HLTh~#ug!fxww8p` zVQvS~p3X>I5E>O(W{JL4K6bCz@vfnJ{Lq(ilLAh$y_m#O!BZka$RkYcTFcxxU4Flt zM7E&6U7?nZ@i}N%VZb&NHu}@7m#Ow7ceT+Ds7NT9uSyqc(6px(CMJg42yF1~IR92UB=zKPTLVEf)b{2mF(T%$^ruahhGvC7>GM{OO^Q#kR#h z)vy<@H8PDT2}8U{F(cm8u#O`Ba>i|NXv`YggAeN6AoI*N(-KOH%?V7Il>bDwl=3On zmwwE_0KGcaOeRG>f7E{5i$C&)hjv%igevbHtRg&7W9r&RUm>N6X)<;!g3dV~i*W+oBjU{yM(^{hb4WsQ2!cZ-!i^j>H zBX%8+1%C-2DJUa}SPLDpPw7pr4yNW_4S2-k@8}E~mY^Z)G>C7QcH;GX<)~FaG4G8z z&ll|xd#xo##G?ac!XHEi!Rvk6{eIgW+IyI5 zI%{HkmkOc9+Q|(9hY_VK&1N{PsfZ3$p+QD=5zdX=xe$X8Wbi~0xsv4 zg=b?%KQ$WV&2zg$meG0!&)}+U7QLCVtIwZw z1Ih<|kc13_Gh~bWKv|!9kV+8YHV^Lm%`EpH=}C zXvEdK_6=LYnMzJFoi;P__efU*bzZM3syr6l=UVtd<_Q?NDn1aH)IG zINtvHDXyQRw9D9!H*P&jEnENqaw}1yIXZbdy1FQO_))#1iOnVrN&;0&3d}TIsdUyX z4+mHcDrr_O!EbWJd%wDg^CMx)?n|S#pHURoX&q4qa!2LTMDW3vyof5hcq}b1J|wT_ zkfDP}Et{_i(LKTsZ-PW+3#;C$y;x}``ec{1+r9olyX}>IXClK1CU_lAy9qcBPED>~OSkpl{yk>3zfC`>vwt)F2e@JSt3v)a!Z&HR zF=n+%>&u@G|1J$T3B7;*Wus4SmU6CU{9h;k+lx1I wH{X0exQDlGf&1eg+zj4atUth`;4Ogvw{!_O(7G=G0C@dTT)&|!c)or81(g}numAu6 literal 0 HcmV?d00001 diff --git a/.yarn/cache/shell-escape-npm-0.2.0-38c5261c29-0d87f1ae22.zip b/.yarn/cache/shell-escape-npm-0.2.0-38c5261c29-0d87f1ae22.zip new file mode 100644 index 0000000000000000000000000000000000000000..af24fa10f54b6f8287a726fe9207f728f14a4a80 GIT binary patch literal 2994 zcmai0dpy%?7+=^BLz$H1dgPLJI7L|`<~p`!E)mAs=6=5nl{7|~TT;lOSU!q0*Rx!@ zxE~ZvQMsIuYTcNnktk<6In_R&cz>VY?|uJxpYQK^pYQv8p9gItC2HMCovq43n6~9699;O{RI(AesyFW01zSo z0Bm2?E(C}7$9WT>e&NAEju>Qci;noMnH}YRr*zY@C+ld*))P9uw(_K{`zRf1gFYBF z%|PJdoPntQ_{RCc`zOA1;)KpkdXK+CU7|3R<^MTke#6t$?~ze2{jJoO*3hnFmGW z_Y0wz<~J|!9ElpO!k8=Ix0q7IXYm^?^`A8Ff=2f`&_)h1ES*E1wx(to%t>yv6>|qC z2M(9yWQLj!juu^@cIkNLxd(0kTvheBPRJ$|4vFw7VC4)Q5bb9~5uyx2z|JDzx6e!5 z)D1LxIO1c`#j#exk}+e^)h#})RF@RP3`%M6JIFckqanlM&9L7)5ws@H^iXG3M` zy=%5~m^vyTR$a_NUp{tzc0O)EPR7#FN_>}jsmD9lz`AXyKipE9+P#%)aZYZQ0vDzf zvth8bFM$8pU13SI4_|tT#}ZHo08m|(C5*X=8PXgYNVuMb43^fFN@9PJfn&y%szjAg z*H!xA#ZFqXg;zFdrW+0Fy&r0xogyt6;rA);Fq6Fc(KC(JIungG*zIGRXhIdN<`Q%n zT_`ChlhfGY!)!|!c^aEKHcRMtPtgaRmLQ3Iq*SL|ZE`scHnob{Me>J32o6MDQ0==S zpu2z?yRAw8AkApA;g$QWa1q$T6H;TMBAMHgQ-%@ppZ-|$P(k+i(JgV|?P1VN5aQ@q z^-R(oAeI|*$9kb4OU2K-qL;pK@^%CLbzn}RS;ov9`)p$}C#3!`xV6$2>)_{xcue2# z()SEjk4=$#thoO@`|plKxE)(`N@ZLsFG7EBg3ZL6cHPXIAIv%|(kv1evDvvrZ;Bc& zRKJ`U-Ma*=+1&`DGF(_F%c@->$5_^O4;HoG1m#on%r=zCI?;~SYin%H5w^9Sa%7Ag zz)|`l3*7|j>y{ek&zkOf0XvePX*5AL1jLm+e$5mk_o?vs9pJs1tMdC}D)Um& z(e5s)RY&S3Ye$kuSF-7mHn0|D@3iy>@<_z+W&_gnWhL}d0dj~Mb{g-GeHzNv??S^p z5@vMAHWaDiHER6|U9zo359-ERbjI^Oa8{Z1vdU13Q$Ir2@qjz=^Zk5xC(wb&33zD0nAG$qv^f3fm z{N-S!9JQz3S)0;%ms&Ll$AmM`WL;{bd$n@s*Ejm(W>=W&Bl{iEy7o<$Um#(*=S?c_1R~E zxm&!RdKyyl@-KV#B#en%Y-Fx83sjZV$}X_R8>2SJJY$LYIUd7@UXek{; zdykThtvO@HfWnJ_T2rQ+w@@ZxzxECCwioHwV%$dd{CqcaTCzswc&GJQvqF3gSzvJ3 za*c?@^q|_5q;Ah?ZohRP3&fVoI5xX3Tj5Oakd`JC{&;f@8wz<8sD@M`v%Nv@d&~_A z*QJ>}cq2W&MgAhIPubhlu<~VT8PtS!nmMoc=NV%Y#kx!D3oVSCJ*#Nv!#-8_ekead zQB4u@U$S?1&0W0Rm9zb|bk1Ardc7duuI?njvCl#Gy3bAnpuPclc)D{4DpAqVwAW?VVOjeP2K62a=B+G z0OQP5y>Kpc^T2tZvp}PFkrVT z)j{vYyaaXjh*!na>4J3oKedw|R3G{DhqG_E+gLJ+^)e8)`$t?A@9@!`kh&%^JSO~Pwp Y{-2d-8z9dE06>)Y^zw-9{o0@Z3o*@#rT_o{ literal 0 HcmV?d00001 diff --git a/package.json b/package.json index b167f077..5adbd9f6 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "pretty-bytes": "^6.1.0", "react": "^18.2.0", "semver": "^7.5.4", + "shell-escape": "^0.2.0", "tempfile": "^5.0.0" }, "devDependencies": { @@ -84,6 +85,7 @@ "@types/pretty-bytes": "^5.2.0", "@types/react": "^18", "@types/semver": "^7.5.0", + "@types/shell-escape": "^0.2.3", "@typescript-eslint/eslint-plugin": "^6.9.1", "@typescript-eslint/parser": "^6.10.0", "@yarnpkg/pnpify": "^4.0.0-rc.48", diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 748d0a7f..7d91c67f 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -17,8 +17,9 @@ import { import { getConnectionDetailsWithPassword } from "../../../lib/database/mysql/connect.js"; import { assertStatus } from "@mittwald/api-client"; import { randomBytes } from "crypto"; -import { executeViaSSH } from "../../../lib/ssh/exec.js"; +import { executeViaSSH, RunCommand } from "../../../lib/ssh/exec.js"; import assertSuccess from "../../../lib/assert_success.js"; +import shellEscape from "shell-escape"; export class Dump extends ExecRenderBaseCommand< typeof Dump, @@ -43,6 +44,14 @@ export class Dump extends ExecRenderBaseCommand< 'The output file to write the dump to. You can specify "-" or "/dev/stdout" to write the dump directly to STDOUT; in this case, you might want to use the --quiet/-q flag to supress all other output, so that you can pipe the mysqldump for further processing.', required: true, }), + gzip: Flags.boolean({ + summary: "compress the dump with gzip", + aliases: ["gz"], + description: + "Compress the dump with gzip. This is useful for large databases, as it can significantly reduce the size of the dump.", + default: false, + required: false, + }), }; static args = { ...mysqlArgs }; @@ -82,6 +91,16 @@ export class Dump extends ExecRenderBaseCommand< const { project } = connectionDetails; const mysqldumpArgs = buildMySqlDumpArgs(connectionDetails); + let cmd: RunCommand = { command: "mysqldump", args: mysqldumpArgs }; + if (this.flags.gzip) { + const escapedArgs = shellEscape(mysqldumpArgs); + cmd = { + shell: `set -xe -o pipefail > /dev/null ; mysqldump ${escapedArgs} | gzip`, + }; + } + + console.log(cmd); + await p.runStep( starting mysqldump via SSH on project {project.shortId} @@ -90,8 +109,7 @@ export class Dump extends ExecRenderBaseCommand< executeViaSSH( this.apiClient, { projectId: connectionDetails.project.id }, - "mysqldump", - mysqldumpArgs, + cmd, this.getOutputStream(), ), ); diff --git a/src/lib/ssh/exec.ts b/src/lib/ssh/exec.ts index dc629d86..2efb3e06 100644 --- a/src/lib/ssh/exec.ts +++ b/src/lib/ssh/exec.ts @@ -6,15 +6,22 @@ import { getSSHConnectionForProject } from "./project.js"; export type RunTarget = { appInstallationId: string } | { projectId: string }; +export type RunCommand = + | { command: string; args: string[] } + | { shell: string }; + export async function executeViaSSH( client: MittwaldAPIV2Client, target: RunTarget, - command: string, - args: string[], + command: RunCommand, output: NodeJS.WritableStream, ): Promise { const { user, host } = await connectionDataForTarget(client, target); - const sshArgs = ["-l", user, "-T", host, command, ...args]; + const sshCommandArgs = + "shell" in command + ? ["bash", "-c", command.shell] + : [command.command, ...command.args]; + const sshArgs = ["-l", user, "-T", host, ...sshCommandArgs]; const ssh = cp.spawn("ssh", sshArgs, { stdio: ["ignore", "pipe", "pipe"], }); diff --git a/yarn.lock b/yarn.lock index 1568aa35..e138d5f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1510,6 +1510,7 @@ __metadata: "@types/pretty-bytes": ^5.2.0 "@types/react": ^18 "@types/semver": ^7.5.0 + "@types/shell-escape": ^0.2.3 "@typescript-eslint/eslint-plugin": ^6.9.1 "@typescript-eslint/parser": ^6.10.0 "@yarnpkg/pnpify": ^4.0.0-rc.48 @@ -1542,6 +1543,7 @@ __metadata: react: ^18.2.0 rimraf: ^5.0.1 semver: ^7.5.4 + shell-escape: ^0.2.0 tempfile: ^5.0.0 type-fest: ^4.6.0 typescript: ^5.1.6 @@ -2950,6 +2952,13 @@ __metadata: languageName: node linkType: hard +"@types/shell-escape@npm:^0.2.3": + version: 0.2.3 + resolution: "@types/shell-escape@npm:0.2.3" + checksum: 05065630b505ab6cfc164021094f23a100a0ffa1758178748b0732da9e3c276209b1d50ffdf53a456fa1cf8063ece53b6e83cef8beec579a0220089707012e72 + languageName: node + linkType: hard + "@types/treeify@npm:^1.0.0": version: 1.0.0 resolution: "@types/treeify@npm:1.0.0" @@ -8911,6 +8920,13 @@ __metadata: languageName: node linkType: hard +"shell-escape@npm:^0.2.0": + version: 0.2.0 + resolution: "shell-escape@npm:0.2.0" + checksum: 0d87f1ae22ad22a74e148348ceaf64721e3024f83c90afcfb527318ce10ece654dd62e103dd89a242f2f4e4ce3cecdef63e3d148c40e5fabca8ba6c508f97d9f + languageName: node + linkType: hard + "shelljs@npm:^0.8.5": version: 0.8.5 resolution: "shelljs@npm:0.8.5" From 74f5b7d74f8d30f4f4d40829abfbb14e6932a956 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Thu, 15 Feb 2024 13:39:58 +0100 Subject: [PATCH 03/16] Update README --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 34447db3..d8cc177b 100644 --- a/README.md +++ b/README.md @@ -2269,7 +2269,7 @@ Create a dump of a MySQL database ``` USAGE - $ mw database mysql dump DATABASE-ID -o [-q] [-p ] [--temporary-user] + $ mw database mysql dump DATABASE-ID -o [-q] [-p ] [--temporary-user] [--gzip] ARGUMENTS DATABASE-ID The ID of the database (when a project context is set, you can also use the name) @@ -2278,6 +2278,7 @@ FLAGS -o, --output= (required) the output file to write the dump to ("-" for stdout) -p, --mysql-password= the password to use for the MySQL user (env: MYSQL_PWD) -q, --quiet suppress process output and only display a machine-readable summary. + --gzip compress the dump with gzip --[no-]temporary-user create a temporary user for the dump FLAG DESCRIPTIONS @@ -2300,6 +2301,11 @@ FLAG DESCRIPTIONS This flag controls if you want to see the process output or only a summary. When using mw non-interactively (e.g. in scripts), you can use this flag to easily get the IDs of created resources for further processing. + --gzip compress the dump with gzip + + Compress the dump with gzip. This is useful for large databases, as it can significantly reduce the size of the + dump. + --[no-]temporary-user create a temporary user for the dump Create a temporary user for the dump. This user will be deleted after the dump has been created. This is useful if From 957caa0df3811c96f38095488bad292aa17d0f3e Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Thu, 15 Feb 2024 13:42:41 +0100 Subject: [PATCH 04/16] Remove "-x" flag from mysqldump shell command --- src/commands/database/mysql/dump.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 7d91c67f..82809bd1 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -95,7 +95,7 @@ export class Dump extends ExecRenderBaseCommand< if (this.flags.gzip) { const escapedArgs = shellEscape(mysqldumpArgs); cmd = { - shell: `set -xe -o pipefail > /dev/null ; mysqldump ${escapedArgs} | gzip`, + shell: `set -e -o pipefail > /dev/null ; mysqldump ${escapedArgs} | gzip`, }; } From f5bbbbba542320d18a952f09797a6afbc9ee3588 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Thu, 15 Feb 2024 14:38:04 +0100 Subject: [PATCH 05/16] Extract temporary MySQL user management into its own function --- src/commands/database/mysql/dump.tsx | 58 ++++---------------------- src/commands/database/mysql/import.tsx | 58 ++++---------------------- src/lib/database/mysql/connect.ts | 39 +++++++++++++++-- src/lib/database/mysql/temp_user.ts | 57 +++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 106 deletions(-) create mode 100644 src/lib/database/mysql/temp_user.ts diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 82809bd1..6b98635f 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -14,11 +14,8 @@ import { mysqlConnectionFlags, withMySQLId, } from "../../../lib/database/mysql/flags.js"; -import { getConnectionDetailsWithPassword } from "../../../lib/database/mysql/connect.js"; -import { assertStatus } from "@mittwald/api-client"; -import { randomBytes } from "crypto"; +import { getConnectionDetailsWithPasswordOrTemporaryUser } from "../../../lib/database/mysql/connect.js"; import { executeViaSSH, RunCommand } from "../../../lib/ssh/exec.js"; -import assertSuccess from "../../../lib/assert_success.js"; import shellEscape from "shell-escape"; export class Dump extends ExecRenderBaseCommand< @@ -64,30 +61,14 @@ export class Dump extends ExecRenderBaseCommand< ); const p = makeProcessRenderer(this.flags, "Dumping a MySQL database"); - const connectionDetails = await getConnectionDetailsWithPassword( - this.apiClient, - databaseId, - p, - this.flags, - ); - - if (this.flags["temporary-user"]) { - const [tempUser, tempPassword] = await p.runStep( - "creating a temporary database user", - () => this.createTemporaryUser(databaseId), + const connectionDetails = + await getConnectionDetailsWithPasswordOrTemporaryUser( + this.apiClient, + databaseId, + p, + this.flags, ); - p.addCleanup("removing temporary database user", async () => { - const r = await this.apiClient.database.deleteMysqlUser({ - mysqlUserId: tempUser.id, - }); - assertSuccess(r); - }); - - connectionDetails.user = tempUser.name; - connectionDetails.password = tempPassword; - } - const { project } = connectionDetails; const mysqldumpArgs = buildMySqlDumpArgs(connectionDetails); @@ -135,31 +116,6 @@ export class Dump extends ExecRenderBaseCommand< return fs.createWriteStream(this.flags.output); } - - private async createTemporaryUser( - databaseId: string, - ): Promise<[{ id: string; name: string }, string]> { - const password = randomBytes(32).toString("base64"); - const createResponse = await this.apiClient.database.createMysqlUser({ - mysqlDatabaseId: databaseId, - data: { - accessLevel: "full", // needed for "PROCESS" privilege - externalAccess: false, - password, - databaseId, - description: "Temporary user for exporting database", - }, - }); - - assertStatus(createResponse, 201); - - const userResponse = await this.apiClient.database.getMysqlUser({ - mysqlUserId: createResponse.data.id, - }); - assertStatus(userResponse, 200); - - return [userResponse.data, password]; - } } function DumpSuccess({ diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index 8e0b5e42..5b1924ef 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -14,11 +14,8 @@ import { mysqlConnectionFlags, withMySQLId, } from "../../../lib/database/mysql/flags.js"; -import { getConnectionDetailsWithPassword } from "../../../lib/database/mysql/connect.js"; -import { assertStatus } from "@mittwald/api-client"; -import { randomBytes } from "crypto"; +import { getConnectionDetailsWithPasswordOrTemporaryUser } from "../../../lib/database/mysql/connect.js"; import { executeViaSSH } from "../../../lib/ssh/exec.js"; -import assertSuccess from "../../../lib/assert_success.js"; export class Import extends ExecRenderBaseCommand< typeof Import, @@ -55,30 +52,14 @@ export class Import extends ExecRenderBaseCommand< ); const p = makeProcessRenderer(this.flags, "Importing a MySQL database"); - const connectionDetails = await getConnectionDetailsWithPassword( - this.apiClient, - databaseId, - p, - this.flags, - ); - - if (this.flags["temporary-user"]) { - const [tempUser, tempPassword] = await p.runStep( - "creating a temporary database user", - () => this.createTemporaryUser(databaseId), + const connectionDetails = + await getConnectionDetailsWithPasswordOrTemporaryUser( + this.apiClient, + databaseId, + p, + this.flags, ); - p.addCleanup("removing temporary database user", async () => { - const r = await this.apiClient.database.deleteMysqlUser({ - mysqlUserId: tempUser.id, - }); - assertSuccess(r); - }); - - connectionDetails.user = tempUser.name; - connectionDetails.password = tempPassword; - } - const { project } = connectionDetails; const mysqlArgs = buildMySqlArgs(connectionDetails); @@ -117,31 +98,6 @@ export class Import extends ExecRenderBaseCommand< return fs.createReadStream(this.flags.input); } - - private async createTemporaryUser( - databaseId: string, - ): Promise<[{ id: string; name: string }, string]> { - const password = randomBytes(32).toString("base64"); - const createResponse = await this.apiClient.database.createMysqlUser({ - mysqlDatabaseId: databaseId, - data: { - accessLevel: "full", // needed for "PROCESS" privilege - externalAccess: false, - password, - databaseId, - description: "Temporary user for exporting database", - }, - }); - - assertStatus(createResponse, 201); - - const userResponse = await this.apiClient.database.getMysqlUser({ - mysqlUserId: createResponse.data.id, - }); - assertStatus(userResponse, 200); - - return [userResponse.data, password]; - } } function ImportSuccess({ diff --git a/src/lib/database/mysql/connect.ts b/src/lib/database/mysql/connect.ts index 3cb62a4b..8fa31e28 100644 --- a/src/lib/database/mysql/connect.ts +++ b/src/lib/database/mysql/connect.ts @@ -4,15 +4,46 @@ import { MittwaldAPIV2, MittwaldAPIV2Client } from "@mittwald/api-client"; import { getProject, getUser } from "../common.js"; import DatabaseMySqlDatabase = MittwaldAPIV2.Components.Schemas.DatabaseMySqlDatabase; import DatabaseMySqlUser = MittwaldAPIV2.Components.Schemas.DatabaseMySqlUser; +import { createTemporaryUser } from "./temp_user.js"; + +export interface MySQLConnectionFlags { + "mysql-password": string | undefined; + "temporary-user"?: boolean; +} + +export async function getConnectionDetailsWithPasswordOrTemporaryUser( + apiClient: MittwaldAPIV2Client, + databaseId: string, + p: ProcessRenderer, + flags: MySQLConnectionFlags, +) { + const connectionDetails = await getConnectionDetailsWithPassword( + apiClient, + databaseId, + p, + flags, + ); + + if (flags["temporary-user"]) { + const { user, password, cleanup } = await p.runStep( + "creating a temporary database user", + () => createTemporaryUser(apiClient, databaseId), + ); + + p.addCleanup("removing temporary database user", cleanup); + + connectionDetails.user = user.name; + connectionDetails.password = password; + } + + return connectionDetails; +} export async function getConnectionDetailsWithPassword( apiClient: MittwaldAPIV2Client, databaseId: string, p: ProcessRenderer, - flags: { - "mysql-password": string | undefined; - "temporary-user"?: boolean; - }, + flags: MySQLConnectionFlags, ) { const password = flags["temporary-user"] ? "" : await getPassword(p, flags); return { diff --git a/src/lib/database/mysql/temp_user.ts b/src/lib/database/mysql/temp_user.ts new file mode 100644 index 00000000..2edd99bc --- /dev/null +++ b/src/lib/database/mysql/temp_user.ts @@ -0,0 +1,57 @@ +import { + assertStatus, + MittwaldAPIV2, + MittwaldAPIV2Client, +} from "@mittwald/api-client"; +import { randomBytes } from "crypto"; +import assertSuccess from "../../assert_success.js"; +import DatabaseMySqlUser = MittwaldAPIV2.Components.Schemas.DatabaseMySqlUser; + +export interface TemporaryUser { + user: DatabaseMySqlUser; + password: string; + + cleanup(): Promise; +} + +/** + * Creates a temporary user for a database operation. + * + * Caution: The returned TemporaryUser object contains a "cleanup()" functions; + * callers of the createTemporaryUser function must make sure to call this + * function (even in case of errors) to reliably clean up any temporary users. + */ +export async function createTemporaryUser( + apiClient: MittwaldAPIV2Client, + databaseId: string, +): Promise { + const password = randomBytes(32).toString("base64"); + const createResponse = await apiClient.database.createMysqlUser({ + mysqlDatabaseId: databaseId, + data: { + accessLevel: "full", // needed for "PROCESS" privilege + externalAccess: false, + password, + databaseId, + description: "Temporary user for exporting/importing database", + }, + }); + + assertStatus(createResponse, 201); + + const userResponse = await apiClient.database.getMysqlUser({ + mysqlUserId: createResponse.data.id, + }); + assertStatus(userResponse, 200); + + return { + user: userResponse.data, + password, + async cleanup() { + const response = await apiClient.database.deleteMysqlUser({ + mysqlUserId: createResponse.data.id, + }); + assertSuccess(response); + }, + }; +} From a721b66e483fc2788659bbea803eaf917981a143 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Thu, 15 Feb 2024 14:39:29 +0100 Subject: [PATCH 06/16] Remove leftover debug statement --- src/commands/database/mysql/dump.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 6b98635f..1488fa90 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -80,8 +80,6 @@ export class Dump extends ExecRenderBaseCommand< }; } - console.log(cmd); - await p.runStep( starting mysqldump via SSH on project {project.shortId} From 59d9c8d82c3220ed2148529d9caf21f0bea3973b Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Thu, 15 Feb 2024 16:12:20 +0100 Subject: [PATCH 07/16] Extract "--[no-]temporary-user" flag --- src/commands/database/mysql/dump.tsx | 11 ++--------- src/commands/database/mysql/import.tsx | 12 ++---------- src/lib/database/mysql/flags.ts | 12 ++++++++++++ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 1488fa90..63003224 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -12,6 +12,7 @@ import { Success } from "../../../rendering/react/components/Success.js"; import { mysqlArgs, mysqlConnectionFlags, + mysqlConnectionFlagsWithTempUser, withMySQLId, } from "../../../lib/database/mysql/flags.js"; import { getConnectionDetailsWithPasswordOrTemporaryUser } from "../../../lib/database/mysql/connect.js"; @@ -25,15 +26,7 @@ export class Dump extends ExecRenderBaseCommand< static summary = "Create a dump of a MySQL database"; static flags = { ...processFlags, - ...mysqlConnectionFlags, - "temporary-user": Flags.boolean({ - summary: "create a temporary user for the dump", - description: - "Create a temporary user for the dump. This user will be deleted after the dump has been created. This is useful if you want to dump a database that is not accessible from the outside.\n\nIf this flag is disabled, you will need to specify the password of the default user; either via the --mysql-password flag or via the MYSQL_PWD environment variable.", - default: true, - required: false, - allowNo: true, - }), + ...mysqlConnectionFlagsWithTempUser, output: Flags.string({ char: "o", summary: 'the output file to write the dump to ("-" for stdout)', diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index 5b1924ef..66b0ac06 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -11,7 +11,7 @@ import * as fs from "fs"; import { Success } from "../../../rendering/react/components/Success.js"; import { mysqlArgs, - mysqlConnectionFlags, + mysqlConnectionFlagsWithTempUser, withMySQLId, } from "../../../lib/database/mysql/flags.js"; import { getConnectionDetailsWithPasswordOrTemporaryUser } from "../../../lib/database/mysql/connect.js"; @@ -24,15 +24,7 @@ export class Import extends ExecRenderBaseCommand< static summary = "Imports a dump of a MySQL database"; static flags = { ...processFlags, - ...mysqlConnectionFlags, - "temporary-user": Flags.boolean({ - summary: "create a temporary user for the dump", - description: - "Create a temporary user for the dump. This user will be deleted after the dump has been imported. This is useful if you want to import a dump into a database that is not accessible from the outside.\n\nIf this flag is disabled, you will need to specify the password of the default user; either via the --mysql-password flag or via the MYSQL_PWD environment variable.", - default: true, - required: false, - allowNo: true, - }), + ...mysqlConnectionFlagsWithTempUser, input: Flags.string({ char: "i", summary: 'the input file from which to read the dump ("-" for stdin)', diff --git a/src/lib/database/mysql/flags.ts b/src/lib/database/mysql/flags.ts index 0d85f86d..5b8fe420 100644 --- a/src/lib/database/mysql/flags.ts +++ b/src/lib/database/mysql/flags.ts @@ -19,6 +19,18 @@ NOTE: This is a security risk, as the password will be visible in the process li }), }; +export const mysqlConnectionFlagsWithTempUser = { + ...mysqlConnectionFlags, + "temporary-user": Flags.boolean({ + summary: "create a temporary user for the dump", + description: + "Create a temporary user for this operation. This user will be deleted after the operation has completed. This is useful if you want to work with a database that is not accessible from the outside.\n\nIf this flag is disabled, you will need to specify the password of the default user; either via the --mysql-password flag or via the MYSQL_PWD environment variable.", + default: true, + required: false, + allowNo: true, + }), +}; + export const mysqlArgs = { "database-id": Args.string({ description: From 7547d66eff0f7e87cd24e38d58c23fc77c9f1836 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Tue, 27 Feb 2024 13:57:49 +0100 Subject: [PATCH 08/16] Add option to import gzipped dump --- src/commands/database/mysql/import.tsx | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index 0f7fdf91..4938d470 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -15,8 +15,9 @@ import { withMySQLId, } from "../../../lib/database/mysql/flags.js"; import { getConnectionDetailsWithPasswordOrTemporaryUser } from "../../../lib/database/mysql/connect.js"; -import { executeViaSSH } from "../../../lib/ssh/exec.js"; +import { executeViaSSH, RunCommand } from "../../../lib/ssh/exec.js"; import { sshConnectionFlags } from "../../../lib/ssh/flags.js"; +import shellEscape from "shell-escape"; export class Import extends ExecRenderBaseCommand< typeof Import, @@ -34,6 +35,14 @@ export class Import extends ExecRenderBaseCommand< 'The input file from which to read the dump to. You can specify "-" or "/dev/stdin" to read the dump directly from STDIN.', required: true, }), + gzip: Flags.boolean({ + summary: "uncompress the dump with gzip", + aliases: ["gz"], + description: + "Uncompress the dump with gzip while importing. This is useful for large databases, as it can significantly reduce the size of the dump.", + default: false, + required: false, + }), }; static args = { ...mysqlArgs }; @@ -57,6 +66,14 @@ export class Import extends ExecRenderBaseCommand< const { project } = connectionDetails; const mysqlArgs = buildMySqlArgs(connectionDetails); + let cmd: RunCommand = { command: "mysql", args: mysqlArgs }; + if (this.flags.gzip) { + const escapedArgs = shellEscape(mysqlArgs); + cmd = { + shell: `set -e -o pipefail > /dev/null ; gunzip | mysqldump ${escapedArgs}`, + }; + } + await p.runStep( starting mysql via SSH on project {project.shortId} @@ -66,7 +83,7 @@ export class Import extends ExecRenderBaseCommand< this.apiClient, this.flags["ssh-user"], { projectId: connectionDetails.project.id }, - { command: "mysql", args: mysqlArgs }, + cmd, null, this.getInputStream(), ), From 70cbf752bf6f4f9347c828eef0321fd40b313efc Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Tue, 27 Feb 2024 15:30:42 +0100 Subject: [PATCH 09/16] Fix documentation --- README.md | 19 +++++++++++++------ src/commands/app/upload.tsx | 34 ++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a4c37e1c..95ce78e6 100644 --- a/README.md +++ b/README.md @@ -1555,17 +1555,18 @@ ARGUMENTS FLAGS -q, --quiet suppress process output and only display a machine-readable summary. - --delete delete local files that are not present on the server - --dry-run do not actually download the app installation + --delete delete remote files that are not present locally + --dry-run do not actually upload the app installation --source= (required) source directory from which to upload the app installation --ssh-user= override the SSH user to connect with; if omitted, your own user will be used DESCRIPTION Upload the filesystem of an app to a project - Upload the filesystem of an app from your local machine to a project.CAUTION: This is a potentially destructive - operation. It will overwrite files on the server with the files from your local machine.This is NOT a turnkey - deployment solution. It is intended for development purposes only. + Upload the filesystem of an app from your local machine to a project. + + CAUTION: This is a potentially destructive operation. It will overwrite files on the server with the files from your + local machine. This is NOT a turnkey deployment solution. It is intended for development purposes only. FLAG DESCRIPTIONS -q, --quiet suppress process output and only display a machine-readable summary. @@ -2409,7 +2410,7 @@ Imports a dump of a MySQL database ``` USAGE - $ mw database mysql import DATABASE-ID -i [-q] [-p ] [--temporary-user] [--ssh-user ] + $ mw database mysql import DATABASE-ID -i [-q] [-p ] [--temporary-user] [--ssh-user ] [--gzip] ARGUMENTS DATABASE-ID The ID of the database (when a project context is set, you can also use the name) @@ -2418,6 +2419,7 @@ FLAGS -i, --input= (required) the input file from which to read the dump ("-" for stdin) -p, --mysql-password= the password to use for the MySQL user (env: MYSQL_PWD) -q, --quiet suppress process output and only display a machine-readable summary. + --gzip uncompress the dump with gzip --ssh-user= override the SSH user to connect with; if omitted, your own user will be used --[no-]temporary-user create a temporary user for the dump @@ -2440,6 +2442,11 @@ FLAG DESCRIPTIONS This flag controls if you want to see the process output or only a summary. When using mw non-interactively (e.g. in scripts), you can use this flag to easily get the IDs of created resources for further processing. + --gzip uncompress the dump with gzip + + Uncompress the dump with gzip while importing. This is useful for large databases, as it can significantly reduce + the size of the dump. + --ssh-user= override the SSH user to connect with; if omitted, your own user will be used This flag can be used to override the SSH user that is used for a connection; be default, your own personal user diff --git a/src/commands/app/upload.tsx b/src/commands/app/upload.tsx index 900a4007..c0fb3787 100644 --- a/src/commands/app/upload.tsx +++ b/src/commands/app/upload.tsx @@ -15,9 +15,9 @@ import { sshConnectionFlags } from "../../lib/ssh/flags.js"; export class Upload extends ExecRenderBaseCommand { static summary = "Upload the filesystem of an app to a project"; static description = - "Upload the filesystem of an app from your local machine to a project." + + "Upload the filesystem of an app from your local machine to a project.\n\n" + "" + - "CAUTION: This is a potentially destructive operation. It will overwrite files on the server with the files from your local machine." + + "CAUTION: This is a potentially destructive operation. It will overwrite files on the server with the files from your local machine. " + "This is NOT a turnkey deployment solution. It is intended for development purposes only."; static args = { ...appInstallationArgs, @@ -26,17 +26,17 @@ export class Upload extends ExecRenderBaseCommand { ...processFlags, ...sshConnectionFlags, "dry-run": Flags.boolean({ - description: "do not actually download the app installation", + description: "do not actually upload the app installation", default: false, }), delete: Flags.boolean({ - description: "delete local files that are not present on the server", + description: "delete remote files that are not present locally", default: false, }), source: Flags.directory({ description: "source directory from which to upload the app installation", required: true, - exists: false, + exists: true, }), }; @@ -89,20 +89,22 @@ export class Upload extends ExecRenderBaseCommand { [...rsyncOpts, source, `${user}@${host}:${directory}/`], ); - if (dryRun) { - await p.complete( - - App would (probably) have successfully been uploaded. 🙂 - , - ); - } else { - await p.complete( - App successfully uploaded; have fun! 🚀, - ); - } + await p.complete(); } protected render(): ReactNode { return undefined; } } + +function UploadSuccess({ dryRun }: { dryRun: boolean }) { + if (dryRun) { + return ( + + App would (probably) have successfully been uploaded. 🙂 + + ); + } + + return App successfully uploaded; have fun! 🚀; +} From 5c7a1086b9e28ef1c30cc0faa583af26ee83a3b0 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Mon, 8 Apr 2024 11:48:32 +0200 Subject: [PATCH 10/16] Fix typo in comment Co-authored-by: LukasFritzeDev <12097997+LukasFritzeDev@users.noreply.github.com> --- src/lib/database/mysql/temp_user.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/database/mysql/temp_user.ts b/src/lib/database/mysql/temp_user.ts index b53a4d5e..4fca8b08 100644 --- a/src/lib/database/mysql/temp_user.ts +++ b/src/lib/database/mysql/temp_user.ts @@ -15,7 +15,7 @@ export interface TemporaryUser { /** * Creates a temporary user for a database operation. * - * Caution: The returned TemporaryUser object contains a "cleanup()" functions; + * Caution: The returned TemporaryUser object contains a "cleanup()" function; * callers of the createTemporaryUser function must make sure to call this * function (even in case of errors) to reliably clean up any temporary users. */ From f2e6a6d9c901b96232481e06b9306de39fc7cf8b Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Mon, 8 Apr 2024 15:45:27 +0200 Subject: [PATCH 11/16] Use "mysql" instead of "mysqldump" --- src/commands/database/mysql/import.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index 5e6c3896..084dbedf 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -65,7 +65,7 @@ export class Import extends ExecRenderBaseCommand< if (this.flags.gzip) { const escapedArgs = shellEscape(mysqlArgs); cmd = { - shell: `set -e -o pipefail > /dev/null ; gunzip | mysqldump ${escapedArgs}`, + shell: `set -e -o pipefail > /dev/null ; gunzip | mysql ${escapedArgs}`, }; } From 981d4dac909df87b1f9a93b2a89bb5d5e883b896 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Mon, 8 Apr 2024 15:51:09 +0200 Subject: [PATCH 12/16] Refactor parameters of "executeViaSSH" function --- src/commands/database/mysql/dump.tsx | 2 +- src/commands/database/mysql/import.tsx | 3 +-- src/lib/ssh/exec.ts | 8 ++++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index ac85a700..29be5d16 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -79,7 +79,7 @@ export class Dump extends ExecRenderBaseCommand< this.flags["ssh-user"], { projectId: connectionDetails.project.id }, cmd, - this.getOutputStream(), + { input: null, output: this.getOutputStream() }, ), ); diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index 084dbedf..82ef9cd1 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -79,8 +79,7 @@ export class Import extends ExecRenderBaseCommand< this.flags["ssh-user"], { projectId: connectionDetails.project.id }, cmd, - null, - this.getInputStream(), + { input: this.getInputStream(), output: null }, ), ); diff --git a/src/lib/ssh/exec.ts b/src/lib/ssh/exec.ts index 0ab26aa1..9859c01e 100644 --- a/src/lib/ssh/exec.ts +++ b/src/lib/ssh/exec.ts @@ -11,13 +11,17 @@ export type RunCommand = | { command: string; args: string[] } | { shell: string }; +export interface RunIO { + input: NodeJS.ReadableStream | null; + output: NodeJS.WritableStream | null; +} + export async function executeViaSSH( client: MittwaldAPIV2Client, sshUser: string | undefined, target: RunTarget, command: RunCommand, - output: NodeJS.WritableStream | null, - input: NodeJS.ReadableStream | null = null, + { input = null, output = null }: RunIO, ): Promise { const { user, host } = await connectionDataForTarget(client, target, sshUser); const sshCommandArgs = From 59ed31530479b6b194bc58adee20e30a391b4ed6 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Mon, 8 Apr 2024 15:59:05 +0200 Subject: [PATCH 13/16] Make construction of mysql password parameter more explicit --- src/commands/database/mysql/dump.tsx | 2 +- src/commands/database/mysql/import.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 29be5d16..0067db10 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -127,5 +127,5 @@ function buildMySqlDumpArgs(d: { password: string; database: string; }): string[] { - return ["-h", d.hostname, "-u", d.user, "-p" + d.password, d.database]; + return ["-h", d.hostname, "-u", d.user, `-p${d.password}`, d.database]; } diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index 82ef9cd1..dcb42218 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -127,5 +127,5 @@ function buildMySqlArgs(d: { password: string; database: string; }): string[] { - return ["-h", d.hostname, "-u", d.user, "-p" + d.password, d.database]; + return ["-h", d.hostname, "-u", d.user, `-p${d.password}`, d.database]; } From d5081206c434800a0948c9291fc3c2efd635f964 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Mon, 8 Apr 2024 17:34:55 +0200 Subject: [PATCH 14/16] Refactor functions for temporary user management --- src/commands/database/mysql/dump.tsx | 66 +++++++++++------------ src/commands/database/mysql/import.tsx | 65 +++++++++++----------- src/lib/database/mysql/connect.ts | 44 ++++++++++----- src/lib/database/mysql/temp_user.ts | 74 +++++++++++++++++--------- src/lib/ssh/exec.ts | 2 +- 5 files changed, 145 insertions(+), 106 deletions(-) diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 0067db10..0a6d6d9b 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -14,7 +14,7 @@ import { mysqlConnectionFlagsWithTempUser, withMySQLId, } from "../../../lib/database/mysql/flags.js"; -import { getConnectionDetailsWithPasswordOrTemporaryUser } from "../../../lib/database/mysql/connect.js"; +import { withConnectionDetails } from "../../../lib/database/mysql/connect.js"; import { executeViaSSH, RunCommand } from "../../../lib/ssh/exec.js"; import shellEscape from "shell-escape"; import { sshConnectionFlags } from "../../../lib/ssh/flags.js"; @@ -50,44 +50,44 @@ export class Dump extends ExecRenderBaseCommand< const databaseId = await withMySQLId(this.apiClient, this.flags, this.args); const p = makeProcessRenderer(this.flags, "Dumping a MySQL database"); - const connectionDetails = - await getConnectionDetailsWithPasswordOrTemporaryUser( - this.apiClient, - databaseId, - p, - this.flags, - ); + const name = await withConnectionDetails( + this.apiClient, + databaseId, + p, + this.flags, + async (connectionDetails) => { + const { project } = connectionDetails; + const mysqldumpArgs = buildMySqlDumpArgs(connectionDetails); - const { project } = connectionDetails; - const mysqldumpArgs = buildMySqlDumpArgs(connectionDetails); + let cmd: RunCommand = { command: "mysqldump", args: mysqldumpArgs }; + if (this.flags.gzip) { + const escapedArgs = shellEscape(mysqldumpArgs); + cmd = { + shell: `set -e -o pipefail > /dev/null ; mysqldump ${escapedArgs} | gzip`, + }; + } - let cmd: RunCommand = { command: "mysqldump", args: mysqldumpArgs }; - if (this.flags.gzip) { - const escapedArgs = shellEscape(mysqldumpArgs); - cmd = { - shell: `set -e -o pipefail > /dev/null ; mysqldump ${escapedArgs} | gzip`, - }; - } + await p.runStep( + + starting mysqldump via SSH on project{" "} + {project.shortId} + , + () => + executeViaSSH( + this.apiClient, + this.flags["ssh-user"], + { projectId: connectionDetails.project.id }, + cmd, + { input: null, output: this.getOutputStream() }, + ), + ); - await p.runStep( - - starting mysqldump via SSH on project {project.shortId} - , - () => - executeViaSSH( - this.apiClient, - this.flags["ssh-user"], - { projectId: connectionDetails.project.id }, - cmd, - { input: null, output: this.getOutputStream() }, - ), + return connectionDetails.database; + }, ); await p.complete( - , + , ); return {}; diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index dcb42218..f119f840 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -14,10 +14,10 @@ import { mysqlConnectionFlagsWithTempUser, withMySQLId, } from "../../../lib/database/mysql/flags.js"; -import { getConnectionDetailsWithPasswordOrTemporaryUser } from "../../../lib/database/mysql/connect.js"; import { executeViaSSH, RunCommand } from "../../../lib/ssh/exec.js"; import { sshConnectionFlags } from "../../../lib/ssh/flags.js"; import shellEscape from "shell-escape"; +import { withConnectionDetails } from "../../../lib/database/mysql/connect.js"; export class Import extends ExecRenderBaseCommand< typeof Import, @@ -50,44 +50,43 @@ export class Import extends ExecRenderBaseCommand< const databaseId = await withMySQLId(this.apiClient, this.flags, this.args); const p = makeProcessRenderer(this.flags, "Importing a MySQL database"); - const connectionDetails = - await getConnectionDetailsWithPasswordOrTemporaryUser( - this.apiClient, - databaseId, - p, - this.flags, - ); + const name = await withConnectionDetails( + this.apiClient, + databaseId, + p, + this.flags, + async (connectionDetails) => { + const { project } = connectionDetails; + const mysqlArgs = buildMySqlArgs(connectionDetails); - const { project } = connectionDetails; - const mysqlArgs = buildMySqlArgs(connectionDetails); + let cmd: RunCommand = { command: "mysql", args: mysqlArgs }; + if (this.flags.gzip) { + const escapedArgs = shellEscape(mysqlArgs); + cmd = { + shell: `set -e -o pipefail > /dev/null ; gunzip | mysql ${escapedArgs}`, + }; + } - let cmd: RunCommand = { command: "mysql", args: mysqlArgs }; - if (this.flags.gzip) { - const escapedArgs = shellEscape(mysqlArgs); - cmd = { - shell: `set -e -o pipefail > /dev/null ; gunzip | mysql ${escapedArgs}`, - }; - } + await p.runStep( + + starting mysql via SSH on project {project.shortId} + , + () => + executeViaSSH( + this.apiClient, + this.flags["ssh-user"], + { projectId: connectionDetails.project.id }, + cmd, + { input: this.getInputStream(), output: null }, + ), + ); - await p.runStep( - - starting mysql via SSH on project {project.shortId} - , - () => - executeViaSSH( - this.apiClient, - this.flags["ssh-user"], - { projectId: connectionDetails.project.id }, - cmd, - { input: this.getInputStream(), output: null }, - ), + return connectionDetails.database; + }, ); await p.complete( - , + , ); return {}; diff --git a/src/lib/database/mysql/connect.ts b/src/lib/database/mysql/connect.ts index bf448d4c..1d8ac12c 100644 --- a/src/lib/database/mysql/connect.ts +++ b/src/lib/database/mysql/connect.ts @@ -4,11 +4,12 @@ import type { MittwaldAPIV2 } from "@mittwald/api-client"; import { MittwaldAPIV2Client } from "@mittwald/api-client"; import { getProject } from "../common.js"; import { getSSHConnectionForProject } from "../../ssh/project.js"; -import { createTemporaryUser } from "./temp_user.js"; +import { withTemporaryUser } from "./temp_user.js"; type DatabaseMySqlDatabase = MittwaldAPIV2.Components.Schemas.DatabaseMySqlDatabase; type DatabaseMySqlUser = MittwaldAPIV2.Components.Schemas.DatabaseMySqlUser; +type Project = MittwaldAPIV2.Components.Schemas.ProjectProject; export interface MySQLConnectionFlags { "mysql-password": string | undefined; @@ -16,12 +17,33 @@ export interface MySQLConnectionFlags { "ssh-user"?: string; } -export async function getConnectionDetailsWithPasswordOrTemporaryUser( +export interface MySQLConnectionDetails { + hostname: string; + database: string; + user: string; + sshHost: string; + sshUser: string; + project: Project; +} + +export type MySQLConnectionDetailsWithPassword = MySQLConnectionDetails & { + password: string; +}; + +/** + * Runs a callback function with connection details for a MySQL database. + * + * Depending on the flags, this function will either use the credentials + * provided in the flags (or prompt for a password), or create a temporary user + * for the operation, which will be cleaned up afterwards. + */ +export async function withConnectionDetails( apiClient: MittwaldAPIV2Client, databaseId: string, p: ProcessRenderer, flags: MySQLConnectionFlags, -) { + cb: (connectionDetails: MySQLConnectionDetailsWithPassword) => Promise, +): Promise { const connectionDetails = await getConnectionDetailsWithPassword( apiClient, databaseId, @@ -30,18 +52,12 @@ export async function getConnectionDetailsWithPasswordOrTemporaryUser( ); if (flags["temporary-user"]) { - const { user, password, cleanup } = await p.runStep( - "creating a temporary database user", - () => createTemporaryUser(apiClient, databaseId), + return withTemporaryUser(apiClient, databaseId, p, async (user, password) => + cb({ ...connectionDetails, user: user.name, password }), ); - - p.addCleanup("removing temporary database user", cleanup); - - connectionDetails.user = user.name; - connectionDetails.password = password; } - return connectionDetails; + return cb(connectionDetails); } export async function getConnectionDetailsWithPassword( @@ -49,7 +65,7 @@ export async function getConnectionDetailsWithPassword( databaseId: string, p: ProcessRenderer, flags: MySQLConnectionFlags, -) { +): Promise { const password = flags["temporary-user"] ? "" : await getPassword(p, flags); const sshUser = flags["ssh-user"]; return { @@ -63,7 +79,7 @@ export async function getConnectionDetails( databaseId: string, sshUser: string | undefined, p: ProcessRenderer, -) { +): Promise { const database = await getDatabase(apiClient, p, databaseId); const databaseUser = await getDatabaseUser(apiClient, p, databaseId); const project = await getProject(apiClient, p, database); diff --git a/src/lib/database/mysql/temp_user.ts b/src/lib/database/mysql/temp_user.ts index 4fca8b08..9c27e228 100644 --- a/src/lib/database/mysql/temp_user.ts +++ b/src/lib/database/mysql/temp_user.ts @@ -2,28 +2,15 @@ import type { MittwaldAPIV2 } from "@mittwald/api-client"; import { assertStatus, MittwaldAPIV2Client } from "@mittwald/api-client"; import { randomBytes } from "crypto"; import assertSuccess from "../../assert_success.js"; +import { ProcessRenderer } from "../../../rendering/process/process.js"; type DatabaseMySqlUser = MittwaldAPIV2.Components.Schemas.DatabaseMySqlUser; -export interface TemporaryUser { - user: DatabaseMySqlUser; - password: string; - - cleanup(): Promise; -} - -/** - * Creates a temporary user for a database operation. - * - * Caution: The returned TemporaryUser object contains a "cleanup()" function; - * callers of the createTemporaryUser function must make sure to call this - * function (even in case of errors) to reliably clean up any temporary users. - */ -export async function createTemporaryUser( +async function createTemporaryUser( apiClient: MittwaldAPIV2Client, databaseId: string, -): Promise { - const password = randomBytes(32).toString("base64"); + password: string, +): Promise { const createResponse = await apiClient.database.createMysqlUser({ mysqlDatabaseId: databaseId, data: { @@ -36,20 +23,57 @@ export async function createTemporaryUser( }); assertStatus(createResponse, 201); + return createResponse.data.id; +} +async function retrieveTemporaryUser( + apiClient: MittwaldAPIV2Client, + mysqlUserId: string, +): Promise { const userResponse = await apiClient.database.getMysqlUser({ - mysqlUserId: createResponse.data.id, + mysqlUserId, }); assertStatus(userResponse, 200); - return { - user: userResponse.data, - password, - async cleanup() { + return userResponse.data; +} + +function generateRandomPassword(): string { + return randomBytes(32).toString("base64"); +} + +/** + * Runs a callback function with a temporary user for a database operation. + * + * Note: This is implemented with a callback parameter, because this function + * also handles the cleanup of the temporary user after the callback has been + * executed. + */ +export async function withTemporaryUser( + apiClient: MittwaldAPIV2Client, + databaseId: string, + p: ProcessRenderer, + cb: (tempUser: DatabaseMySqlUser, password: string) => Promise, +): Promise { + const password = generateRandomPassword(); + const user = await p.runStep("creating temporary user", async () => { + const mysqlUserId = await createTemporaryUser( + apiClient, + databaseId, + password, + ); + + return await retrieveTemporaryUser(apiClient, mysqlUserId); + }); + + try { + return await cb(user, password); + } finally { + await p.runStep("removing temporary user", async () => { const response = await apiClient.database.deleteMysqlUser({ - mysqlUserId: createResponse.data.id, + mysqlUserId: user.id, }); assertSuccess(response); - }, - }; + }); + } } diff --git a/src/lib/ssh/exec.ts b/src/lib/ssh/exec.ts index 9859c01e..4c1e16a9 100644 --- a/src/lib/ssh/exec.ts +++ b/src/lib/ssh/exec.ts @@ -53,7 +53,7 @@ export async function executeViaSSH( if (code === 0) { res(undefined); } else { - rej(new Error(`ssh+${command} exited with code ${code}\n${err}`)); + rej(new Error(`command exited with code ${code}\n${err}`)); } }; From bcef63d3954d3995131d6dc1b1cc3cbb6b66a802 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Tue, 9 Apr 2024 14:30:52 +0200 Subject: [PATCH 15/16] Make variable names more clear --- src/commands/database/mysql/dump.tsx | 4 ++-- src/commands/database/mysql/import.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 0a6d6d9b..5039e944 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -50,7 +50,7 @@ export class Dump extends ExecRenderBaseCommand< const databaseId = await withMySQLId(this.apiClient, this.flags, this.args); const p = makeProcessRenderer(this.flags, "Dumping a MySQL database"); - const name = await withConnectionDetails( + const databaseName = await withConnectionDetails( this.apiClient, databaseId, p, @@ -87,7 +87,7 @@ export class Dump extends ExecRenderBaseCommand< ); await p.complete( - , + , ); return {}; diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index f119f840..e804fc95 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -50,7 +50,7 @@ export class Import extends ExecRenderBaseCommand< const databaseId = await withMySQLId(this.apiClient, this.flags, this.args); const p = makeProcessRenderer(this.flags, "Importing a MySQL database"); - const name = await withConnectionDetails( + const databaseName = await withConnectionDetails( this.apiClient, databaseId, p, @@ -86,7 +86,7 @@ export class Import extends ExecRenderBaseCommand< ); await p.complete( - , + , ); return {}; From c2b21c04ef61497d6105ef83190dc26d299b004b Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Tue, 9 Apr 2024 14:31:24 +0200 Subject: [PATCH 16/16] Rename "{with => runWith}ConnectionDetails" for clarity --- src/commands/database/mysql/dump.tsx | 4 ++-- src/commands/database/mysql/import.tsx | 4 ++-- src/lib/database/mysql/connect.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/commands/database/mysql/dump.tsx b/src/commands/database/mysql/dump.tsx index 5039e944..2641a668 100644 --- a/src/commands/database/mysql/dump.tsx +++ b/src/commands/database/mysql/dump.tsx @@ -14,7 +14,7 @@ import { mysqlConnectionFlagsWithTempUser, withMySQLId, } from "../../../lib/database/mysql/flags.js"; -import { withConnectionDetails } from "../../../lib/database/mysql/connect.js"; +import { runWithConnectionDetails } from "../../../lib/database/mysql/connect.js"; import { executeViaSSH, RunCommand } from "../../../lib/ssh/exec.js"; import shellEscape from "shell-escape"; import { sshConnectionFlags } from "../../../lib/ssh/flags.js"; @@ -50,7 +50,7 @@ export class Dump extends ExecRenderBaseCommand< const databaseId = await withMySQLId(this.apiClient, this.flags, this.args); const p = makeProcessRenderer(this.flags, "Dumping a MySQL database"); - const databaseName = await withConnectionDetails( + const databaseName = await runWithConnectionDetails( this.apiClient, databaseId, p, diff --git a/src/commands/database/mysql/import.tsx b/src/commands/database/mysql/import.tsx index e804fc95..1e812fc3 100644 --- a/src/commands/database/mysql/import.tsx +++ b/src/commands/database/mysql/import.tsx @@ -17,7 +17,7 @@ import { import { executeViaSSH, RunCommand } from "../../../lib/ssh/exec.js"; import { sshConnectionFlags } from "../../../lib/ssh/flags.js"; import shellEscape from "shell-escape"; -import { withConnectionDetails } from "../../../lib/database/mysql/connect.js"; +import { runWithConnectionDetails } from "../../../lib/database/mysql/connect.js"; export class Import extends ExecRenderBaseCommand< typeof Import, @@ -50,7 +50,7 @@ export class Import extends ExecRenderBaseCommand< const databaseId = await withMySQLId(this.apiClient, this.flags, this.args); const p = makeProcessRenderer(this.flags, "Importing a MySQL database"); - const databaseName = await withConnectionDetails( + const databaseName = await runWithConnectionDetails( this.apiClient, databaseId, p, diff --git a/src/lib/database/mysql/connect.ts b/src/lib/database/mysql/connect.ts index 1d8ac12c..8086cf39 100644 --- a/src/lib/database/mysql/connect.ts +++ b/src/lib/database/mysql/connect.ts @@ -37,7 +37,7 @@ export type MySQLConnectionDetailsWithPassword = MySQLConnectionDetails & { * provided in the flags (or prompt for a password), or create a temporary user * for the operation, which will be cleaned up afterwards. */ -export async function withConnectionDetails( +export async function runWithConnectionDetails( apiClient: MittwaldAPIV2Client, databaseId: string, p: ProcessRenderer,