diff --git a/README.md b/README.md index 770ec49a..80208219 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ $ npm install -g kourou $ kourou COMMAND running command... $ kourou (-v|--version|version) -kourou/0.27.1 darwin-arm64 node-v18.17.1 +kourou/0.28.0 darwin-arm64 node-v20.10.0 $ kourou --help [COMMAND] USAGE $ kourou COMMAND @@ -161,6 +161,7 @@ All other arguments and options will be passed as-is to the `sdk:query` method. * [`kourou instance:logs`](#kourou-instancelogs) * [`kourou instance:spawn`](#kourou-instancespawn) * [`kourou paas:deploy ENVIRONMENT APPLICATIONID IMAGE`](#kourou-paasdeploy-environment-applicationid-image) +* [`kourou paas:elasticsearch:dump ENVIRONMENT APPLICATIONID DUMPDIRECTORY`](#kourou-paaselasticsearchdump-environment-applicationid-dumpdirectory) * [`kourou paas:init PROJECT`](#kourou-paasinit-project) * [`kourou paas:login`](#kourou-paaslogin) * [`kourou paas:logs ENVIRONMENT APPLICATION`](#kourou-paaslogs-environment-application) @@ -1064,6 +1065,25 @@ OPTIONS _See code: [lib/commands/paas/deploy.js](lib/commands/paas/deploy.js)_ +## `kourou paas:elasticsearch:dump ENVIRONMENT APPLICATIONID DUMPDIRECTORY` + +Dump data from the Elasticsearch of a PaaS application + +``` +USAGE + $ kourou paas:elasticsearch:dump ENVIRONMENT APPLICATIONID DUMPDIRECTORY + +ARGUMENTS + ENVIRONMENT Project environment name + APPLICATIONID Application Identifier + DUMPDIRECTORY Directory where to store dump files + +OPTIONS + --batch-size=batch-size [default: 2000] Maximum batch size + --help show CLI help + --project=project Current PaaS project +``` + ## `kourou paas:init PROJECT` Initialize a PaaS project in current directory diff --git a/package-lock.json b/package-lock.json index 773589e4..f66df273 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "kourou", - "version": "0.27.1", + "version": "0.28.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "kourou", - "version": "0.27.1", + "version": "0.28.0", "license": "Apache-2.0", "dependencies": { "@elastic/elasticsearch": "^7.12.0", @@ -49,7 +49,7 @@ "@types/listr": "^0.14.2", "@types/mocha": "^8.2.2", "@types/ndjson": "^2.0.0", - "@types/node": "^14.14.41", + "@types/node": "^18.19.0", "@types/node-emoji": "^1.8.1", "@types/node-fetch": "^2.6.1", "@types/tar": "^6.1.3", @@ -1367,10 +1367,13 @@ } }, "node_modules/@types/node": { - "version": "14.18.36", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.36.tgz", - "integrity": "sha512-FXKWbsJ6a1hIrRxv+FoukuHnGTgEzKYGi7kilfMae96AL9UNkPFNWJEEYWzdRI9ooIkbr4AKldyuSTLql06vLQ==", - "dev": true + "version": "18.19.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.4.tgz", + "integrity": "sha512-xNzlUhzoHotIsnFoXmJB+yWmBvFZgKCI9TtPIEdYIMM1KWfwuY8zh7wvc1u1OAXlC7dlf6mZVx/s+Y5KfFz19A==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/node-emoji": { "version": "1.8.2", @@ -7957,6 +7960,12 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "node_modules/universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -9490,10 +9499,13 @@ } }, "@types/node": { - "version": "14.18.36", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.36.tgz", - "integrity": "sha512-FXKWbsJ6a1hIrRxv+FoukuHnGTgEzKYGi7kilfMae96AL9UNkPFNWJEEYWzdRI9ooIkbr4AKldyuSTLql06vLQ==", - "dev": true + "version": "18.19.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.4.tgz", + "integrity": "sha512-xNzlUhzoHotIsnFoXmJB+yWmBvFZgKCI9TtPIEdYIMM1KWfwuY8zh7wvc1u1OAXlC7dlf6mZVx/s+Y5KfFz19A==", + "dev": true, + "requires": { + "undici-types": "~5.26.4" + } }, "@types/node-emoji": { "version": "1.8.2", @@ -14450,6 +14462,12 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" }, + "undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", diff --git a/package.json b/package.json index 829fb6ac..82963ad3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "kourou", "description": "The CLI that helps you manage your Kuzzle instances", - "version": "0.27.1", + "version": "0.28.0", "author": "The Kuzzle Team ", "bin": { "kourou": "./bin/run" @@ -56,7 +56,7 @@ "@types/listr": "^0.14.2", "@types/mocha": "^8.2.2", "@types/ndjson": "^2.0.0", - "@types/node": "^14.14.41", + "@types/node": "^18.19.0", "@types/node-emoji": "^1.8.1", "@types/node-fetch": "^2.6.1", "@types/tar": "^6.1.3", diff --git a/src/commands/paas/elasticsearch/dump.ts b/src/commands/paas/elasticsearch/dump.ts new file mode 100644 index 00000000..c73266ed --- /dev/null +++ b/src/commands/paas/elasticsearch/dump.ts @@ -0,0 +1,226 @@ +import path from "path"; +import fs from "node:fs/promises"; + +import ndjson from "ndjson"; +import { flags } from "@oclif/command"; + +import { PaasKommand } from "../../../support/PaasKommand"; + +/** + * Results of the document dump action. + */ +type DocumentDump = { + pit_id: string; + hits: DocumentDumpHits; +}; + +type DocumentDumpHits = { + total: DocumentDumpHitsTotal; + hits: DocumentDumpHit[]; +}; + +type DocumentDumpHitsTotal = { + value: number; +}; + +type DocumentDumpHit = { + sort: string[]; +}; + +class PaasEsDump extends PaasKommand { + public static description = "Dump data from the Elasticsearch of a PaaS application"; + + public static flags = { + help: flags.help(), + project: flags.string({ + description: "Current PaaS project", + }), + "batch-size": flags.integer({ + description: "Maximum batch size", + default: 2000, + }), + }; + + static args = [ + { + name: "environment", + description: "Project environment name", + required: true, + }, + { + name: "applicationId", + description: "Application Identifier", + required: true, + }, + { + name: "dumpDirectory", + description: "Directory where to store dump files", + required: true, + } + ]; + + async runSafe() { + // Check that the batch size is positive + if (this.flags["batch-size"] <= 0) { + this.logKo(`The batch size must be greater than zero. (Specified batch size: ${this.flags["batch-size"]})`); + process.exit(1); + } + + // Log in to the PaaS + const apiKey = await this.getCredentials(); + + await this.initPaasClient({ apiKey }); + + const user = await this.paas.auth.getCurrentUser(); + this.logInfo( + `Logged as "${user._id}" for project "${this.flags.project || this.getProject() + }"` + ); + + // Create the dump directory + await fs.mkdir(this.args.dumpDirectory, { recursive: true }); + + // Dump the indexes + this.logInfo("Dumping Elasticsearch indexes..."); + + const indexesResult = await this.getAllIndexes(); + await fs.writeFile(path.join(this.args.dumpDirectory, "indexes.json"), JSON.stringify(indexesResult)); + + this.logOk("Elasticsearch indexes dumped!"); + + // Dump all the documents + this.logInfo("Dumping Elasticsearch documents..."); + await this.dumpAllDocuments(); + + this.logOk("Elasticsearch documents dumped!"); + this.logOk(`The dumped files are available under "${path.resolve(this.args.dumpDirectory)}"`) + } + + /** + * @description Get all indexes from the Elasticsearch of the PaaS application. + * @returns The indexes. + */ + private async getAllIndexes() { + const { result }: any = await this.paas.query({ + controller: "application/storage", + action: "getIndexes", + environmentId: this.args.environment, + projectId: this.flags.project || this.getProject(), + applicationId: this.args.applicationId, + body: {}, + }); + + return result; + } + + /** + * @description Dump documents from the Elasticsearch of the PaaS application. + * @param pitId ID of the PIT opened on Elasticsearch. + * @param searchAfter Cursor for dumping documents after a certain one. + * @returns The dumped documents. + */ + private async dumpDocuments(pitId: string, searchAfter: string[]): Promise { + const { result }: any = await this.paas.query({ + controller: "application/storage", + action: "dumpDocuments", + environmentId: this.args.environment, + projectId: this.flags.project || this.getProject(), + applicationId: this.args.applicationId, + body: { + pitId, + searchAfter: JSON.stringify(searchAfter), + size: this.flags["batch-size"], + }, + }); + + return result; + } + + private async dumpAllDocuments() { + // Prepare dumping all documents + let pitId = ""; + let searchAfter: string[] = []; + + let dumpedDocuments = 0; + let totalDocuments = 0; + + const fd = await fs.open(path.join(this.args.dumpDirectory, "documents.jsonl"), "w"); + const writeStream = fd.createWriteStream(); + const ndjsonStream = ndjson.stringify(); + + writeStream.on("error", (error) => { + throw error; + }); + + ndjsonStream.on("data", (line: string) => { + writeStream.write(line); + }); + + const teardown = async () => { + // Finish the dump session if a PIT ID is set + if (pitId.length > 0) { + await this.finishDump(pitId); + } + + // Close the open streams/file + writeStream.close(); + await fd.close(); + }; + + try { + // Dump the first batch + let result = await this.dumpDocuments(pitId, searchAfter); + let hits = result.hits.hits; + + while (hits.length > 0) { + // Update the PIT ID and the cursor for the next dump + pitId = result.pit_id; + searchAfter = hits[hits.length - 1].sort; + + // Save the documents + for (let i = 0; i < hits.length; ++i) { + ndjsonStream.write(hits[i]); + } + + dumpedDocuments += hits.length; + totalDocuments = result.hits.total.value; + this.logInfo(`Dumping Elasticsearch documents: ${Math.floor(dumpedDocuments / totalDocuments * 100)}% (${dumpedDocuments}/${totalDocuments})`); + + // Dump the next batch + result = await this.dumpDocuments(pitId, searchAfter); + hits = result.hits.hits; + } + } catch (error: any) { + teardown(); + + this.logKo(`Error while dumping the documents: ${error}`); + process.exit(1); + } + + // Finish the dump + teardown(); + } + + /** + * @description Finish the document dumping session. + * @param pitId ID of the PIT opened on Elasticsearch. + */ + private async finishDump(pitId: string) { + try { + await this.paas.query({ + controller: "application/storage", + action: "finishDumpDocuments", + environmentId: this.args.environment, + projectId: this.flags.project || this.getProject(), + applicationId: this.args.applicationId, + body: { + pitId, + }, + }); + } catch (error: any) { + this.logInfo(`Unable to cleanly finish the dump session: ${error}`); + } + } +} + +export default PaasEsDump; diff --git a/src/commands/paas/snapshots/restore.ts b/src/commands/paas/snapshots/restore.ts index f52f0405..64be1569 100644 --- a/src/commands/paas/snapshots/restore.ts +++ b/src/commands/paas/snapshots/restore.ts @@ -55,10 +55,7 @@ class PaasSnapshotsRestore extends PaasKommand { environmentId: this.args.environment, projectId: this.flags.project || this.getProject(), applicationId: this.args.applicationId, - body: { - repository: "automated", - snapshot: this.args.snapshotId, - }, + snapshotId: this.args.snapshotId }); this.logInfo("Ok");