From 20c5fba92bcd24df1a53bdfc1ec8489d78c48403 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Sun, 17 Sep 2023 12:16:52 +0800 Subject: [PATCH] feat(nodejs): Add Node.js Implementation of Apache RocketMQ Client (#602) * feat(nodejs): Add Node.js Implementation of Apache RocketMQ Client * fix MessageQueue * add default file logger to `$HOME/logs/rocketmq/rocketmq_client_nodejs.log` * cleanup timers on shutdown * Change log format * Add delay message unittest * Add sessionCredentials * Run test on GitHub Action * Fix coverage yml * Should init before start test * Checkout with submodules=recursive * Show cluster stats before create topic * Print receive stats on unittest * Use random consumerGroup on receive * Set coverage files * Set version to 1.0.0 * All message type support * Add FIFO message test * Make sure FIFO message test pass * Add LICENSE to npm package * Default max number of message sending retries is 3 --- .github/workflows/build.yml | 10 +- .github/workflows/nodejs_build.yml | 26 + .github/workflows/nodejs_coverage.yml | 46 ++ .licenserc.yaml | 2 + docs/observability.md | 13 +- nodejs/.eslintrc | 6 + nodejs/.gitignore | 7 + nodejs/LICENSE | 201 +++++++ nodejs/README-CN.md | 101 ++++ nodejs/README.md | 100 ++++ nodejs/examples/Producer.cjs | 42 ++ nodejs/examples/Producer.ts | 33 + nodejs/examples/SimpleConsumer.cjs | 39 ++ nodejs/examples/SimpleConsumer.ts | 33 + nodejs/package.json | 61 ++ nodejs/proto/apache/rocketmq/v2/admin.proto | 43 ++ .../proto/apache/rocketmq/v2/definition.proto | 564 ++++++++++++++++++ nodejs/proto/apache/rocketmq/v2/service.proto | 411 +++++++++++++ nodejs/scripts/build-grpc.sh | 71 +++ nodejs/src/client/BaseClient.ts | 384 ++++++++++++ nodejs/src/client/ClientId.ts | 31 + nodejs/src/client/Logger.ts | 35 ++ nodejs/src/client/RpcClient.ts | 245 ++++++++ nodejs/src/client/RpcClientManager.ts | 173 ++++++ nodejs/src/client/SessionCredentials.ts | 22 + nodejs/src/client/Settings.ts | 45 ++ nodejs/src/client/TelemetrySession.ts | 118 ++++ nodejs/src/client/UserAgent.ts | 45 ++ nodejs/src/client/index.ts | 27 + nodejs/src/consumer/Consumer.ts | 111 ++++ nodejs/src/consumer/FilterExpression.ts | 42 ++ nodejs/src/consumer/SimpleConsumer.ts | 140 +++++ .../consumer/SimpleSubscriptionSettings.ts | 65 ++ .../src/consumer/SubscriptionLoadBalancer.ts | 47 ++ nodejs/src/consumer/index.ts | 22 + nodejs/src/exception/BadRequestException.ts | 25 + nodejs/src/exception/ClientException.ts | 29 + nodejs/src/exception/ForbiddenException.ts | 25 + .../src/exception/InternalErrorException.ts | 25 + nodejs/src/exception/NotFoundException.ts | 25 + .../src/exception/PayloadTooLargeException.ts | 25 + .../src/exception/PaymentRequiredException.ts | 25 + nodejs/src/exception/ProxyTimeoutException.ts | 25 + .../RequestHeaderFieldsTooLargeException.ts | 25 + nodejs/src/exception/StatusChecker.ts | 94 +++ .../src/exception/TooManyRequestsException.ts | 25 + nodejs/src/exception/UnauthorizedException.ts | 25 + nodejs/src/exception/UnsupportedException.ts | 25 + nodejs/src/exception/index.ts | 30 + nodejs/src/index.ts | 23 + nodejs/src/message/Message.ts | 51 ++ nodejs/src/message/MessageId.ts | 123 ++++ nodejs/src/message/MessageView.ts | 94 +++ nodejs/src/message/PublishingMessage.ts | 102 ++++ nodejs/src/message/index.ts | 21 + nodejs/src/producer/Producer.ts | 291 +++++++++ nodejs/src/producer/PublishingLoadBalancer.ts | 85 +++ nodejs/src/producer/PublishingSettings.ts | 79 +++ nodejs/src/producer/SendReceipt.ts | 60 ++ nodejs/src/producer/Transaction.ts | 70 +++ nodejs/src/producer/TransactionChecker.ts | 23 + nodejs/src/producer/index.ts | 23 + .../retry/ExponentialBackoffRetryPolicy.ts | 76 +++ nodejs/src/retry/RetryPolicy.ts | 51 ++ nodejs/src/retry/index.ts | 19 + nodejs/src/route/Broker.ts | 39 ++ nodejs/src/route/Endpoints.ts | 70 +++ nodejs/src/route/MessageQueue.ts | 49 ++ nodejs/src/route/TopicRouteData.ts | 38 ++ nodejs/src/route/index.ts | 21 + nodejs/src/util/index.ts | 83 +++ nodejs/test/consumer/SimpleConsumer.test.ts | 138 +++++ nodejs/test/helper.ts | 36 ++ nodejs/test/index.test.ts | 31 + nodejs/test/message/MessageId.test.ts | 49 ++ nodejs/test/producer/Producer.test.ts | 283 +++++++++ nodejs/test/start-rocketmq.sh | 40 ++ nodejs/test/util/index.test.ts | 39 ++ nodejs/tsconfig.json | 11 + nodejs/tsconfig.prod.json | 4 + 80 files changed, 5904 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/nodejs_build.yml create mode 100644 .github/workflows/nodejs_coverage.yml create mode 100644 nodejs/.eslintrc create mode 100644 nodejs/.gitignore create mode 100644 nodejs/LICENSE create mode 100644 nodejs/README-CN.md create mode 100644 nodejs/README.md create mode 100644 nodejs/examples/Producer.cjs create mode 100644 nodejs/examples/Producer.ts create mode 100644 nodejs/examples/SimpleConsumer.cjs create mode 100644 nodejs/examples/SimpleConsumer.ts create mode 100644 nodejs/package.json create mode 100644 nodejs/proto/apache/rocketmq/v2/admin.proto create mode 100644 nodejs/proto/apache/rocketmq/v2/definition.proto create mode 100644 nodejs/proto/apache/rocketmq/v2/service.proto create mode 100755 nodejs/scripts/build-grpc.sh create mode 100644 nodejs/src/client/BaseClient.ts create mode 100644 nodejs/src/client/ClientId.ts create mode 100644 nodejs/src/client/Logger.ts create mode 100644 nodejs/src/client/RpcClient.ts create mode 100644 nodejs/src/client/RpcClientManager.ts create mode 100644 nodejs/src/client/SessionCredentials.ts create mode 100644 nodejs/src/client/Settings.ts create mode 100644 nodejs/src/client/TelemetrySession.ts create mode 100644 nodejs/src/client/UserAgent.ts create mode 100644 nodejs/src/client/index.ts create mode 100644 nodejs/src/consumer/Consumer.ts create mode 100644 nodejs/src/consumer/FilterExpression.ts create mode 100644 nodejs/src/consumer/SimpleConsumer.ts create mode 100644 nodejs/src/consumer/SimpleSubscriptionSettings.ts create mode 100644 nodejs/src/consumer/SubscriptionLoadBalancer.ts create mode 100644 nodejs/src/consumer/index.ts create mode 100644 nodejs/src/exception/BadRequestException.ts create mode 100644 nodejs/src/exception/ClientException.ts create mode 100644 nodejs/src/exception/ForbiddenException.ts create mode 100644 nodejs/src/exception/InternalErrorException.ts create mode 100644 nodejs/src/exception/NotFoundException.ts create mode 100644 nodejs/src/exception/PayloadTooLargeException.ts create mode 100644 nodejs/src/exception/PaymentRequiredException.ts create mode 100644 nodejs/src/exception/ProxyTimeoutException.ts create mode 100644 nodejs/src/exception/RequestHeaderFieldsTooLargeException.ts create mode 100644 nodejs/src/exception/StatusChecker.ts create mode 100644 nodejs/src/exception/TooManyRequestsException.ts create mode 100644 nodejs/src/exception/UnauthorizedException.ts create mode 100644 nodejs/src/exception/UnsupportedException.ts create mode 100644 nodejs/src/exception/index.ts create mode 100644 nodejs/src/index.ts create mode 100644 nodejs/src/message/Message.ts create mode 100644 nodejs/src/message/MessageId.ts create mode 100644 nodejs/src/message/MessageView.ts create mode 100644 nodejs/src/message/PublishingMessage.ts create mode 100644 nodejs/src/message/index.ts create mode 100644 nodejs/src/producer/Producer.ts create mode 100644 nodejs/src/producer/PublishingLoadBalancer.ts create mode 100644 nodejs/src/producer/PublishingSettings.ts create mode 100644 nodejs/src/producer/SendReceipt.ts create mode 100644 nodejs/src/producer/Transaction.ts create mode 100644 nodejs/src/producer/TransactionChecker.ts create mode 100644 nodejs/src/producer/index.ts create mode 100644 nodejs/src/retry/ExponentialBackoffRetryPolicy.ts create mode 100644 nodejs/src/retry/RetryPolicy.ts create mode 100644 nodejs/src/retry/index.ts create mode 100644 nodejs/src/route/Broker.ts create mode 100644 nodejs/src/route/Endpoints.ts create mode 100644 nodejs/src/route/MessageQueue.ts create mode 100644 nodejs/src/route/TopicRouteData.ts create mode 100644 nodejs/src/route/index.ts create mode 100644 nodejs/src/util/index.ts create mode 100644 nodejs/test/consumer/SimpleConsumer.test.ts create mode 100644 nodejs/test/helper.ts create mode 100644 nodejs/test/index.test.ts create mode 100644 nodejs/test/message/MessageId.test.ts create mode 100644 nodejs/test/producer/Producer.test.ts create mode 100755 nodejs/test/start-rocketmq.sh create mode 100644 nodejs/test/util/index.test.ts create mode 100644 nodejs/tsconfig.json create mode 100644 nodejs/tsconfig.prod.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99b8910b8..5c4053023 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,7 @@ jobs: php: ${{ steps.filter.outputs.php }} rust: ${{ steps.filter.outputs.rust }} python: ${{ steps.filter.outputs.python }} + nodejs: ${{ steps.filter.outputs.nodejs }} steps: - uses: actions/checkout@v2 - uses: dorny/paths-filter@v2 @@ -37,6 +38,8 @@ jobs: - 'rust/**' python: - 'python/**' + nodejs: + - 'nodejs/**' java-build: needs: [paths-filter] if: ${{ needs.paths-filter.outputs.java == 'true' }} @@ -66,9 +69,13 @@ jobs: needs: [paths-filter] if: ${{ needs.paths-filter.outputs.python == 'true' }} uses: ./.github/workflows/python_build.yml + nodejs-build: + needs: [paths-filter] + if: ${{ needs.paths-filter.outputs.nodejs == 'true' }} + uses: ./.github/workflows/nodejs_build.yml build-result: runs-on: ubuntu-latest - needs: [java-build, cpp-build, csharp-build, golang-build, php-build, rust-build, python-build] + needs: [java-build, cpp-build, csharp-build, golang-build, php-build, rust-build, python-build, nodejs-build] if: ${{ always() }} steps: - uses: actions/checkout@v2 @@ -80,6 +87,7 @@ jobs: golang-${{ needs.golang-build.result }},\ php-${{ needs.php-build.result }},\ rust-${{ needs.rust-build.result }},\ + nodejs-${{ needs.nodejs-build.result }},\ python-${{ needs.python-build.result }} | grep -E 'cancelled|failure' -o > null then echo "There are failed/cancelled builds" diff --git a/.github/workflows/nodejs_build.yml b/.github/workflows/nodejs_build.yml new file mode 100644 index 000000000..f8b37b71b --- /dev/null +++ b/.github/workflows/nodejs_build.yml @@ -0,0 +1,26 @@ +name: Node.js Build +on: + workflow_call: +jobs: + build: + name: "${{ matrix.os }}, node-${{ matrix.nodejs }}" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + nodejs: [18] + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Set up Node.js ${{ matrix.nodejs }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.nodejs }} + + - name: Build + working-directory: ./nodejs + run: npm i && npm run init && npm run build && npm pack diff --git a/.github/workflows/nodejs_coverage.yml b/.github/workflows/nodejs_coverage.yml new file mode 100644 index 000000000..2a29a5b28 --- /dev/null +++ b/.github/workflows/nodejs_coverage.yml @@ -0,0 +1,46 @@ +name: Node.js Coverage +on: + pull_request: + types: [opened, reopened, synchronize] + paths: + - 'nodejs/**' + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [16.19.0, 16.x, 18.x, 20.x] + steps: + - name: Checkout Git Source + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + - name: Install dependencies + working-directory: ./nodejs + run: npm i && npm run init + + - name: Start RocketMQ Server + working-directory: ./nodejs + run: npm run start-rocketmq + + - name: Run test + working-directory: ./nodejs + run: npm run ci + + - name: Code Coverage + uses: codecov/codecov-action@v3 + with: + files: ./nodejs/coverage/coverage-final.json + flags: nodejs + fail_ci_if_error: true + verbose: true diff --git a/.licenserc.yaml b/.licenserc.yaml index 9e97dead3..4edcd18fc 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -52,5 +52,7 @@ header: - 'php/composer.json' - 'rust/.cargo/Cargo.lock.min' - 'rust/src/pb/*.rs' + - 'nodejs/**/*.json' + - 'nodejs/.eslintrc' comment: on-failure diff --git a/docs/observability.md b/docs/observability.md index d94090aa1..40ea777ce 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -8,9 +8,10 @@ ### Logging Default Path -| Language | Path | -| -------- | ---------------------------------------------| -| CPP | `$HOME/logs/rocketmq/client.log` | -| C# | `$HOME/logs/rocketmq/rocketmq-client.log` | -| Java | `$HOME/logs/rocketmq/rocketmq-client.log` | -| Go | `$HOME/logs/rocketmq/rocketmq_client_go.log` | +| Language | Path | +| -------- | -------------------------------------------------| +| CPP | `$HOME/logs/rocketmq/client.log` | +| C# | `$HOME/logs/rocketmq/rocketmq-client.log` | +| Java | `$HOME/logs/rocketmq/rocketmq-client.log` | +| Go | `$HOME/logs/rocketmq/rocketmq_client_go.log` | +| Node.js | `$HOME/logs/rocketmq/rocketmq_client_nodejs.log` | diff --git a/nodejs/.eslintrc b/nodejs/.eslintrc new file mode 100644 index 000000000..9bcdb4688 --- /dev/null +++ b/nodejs/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": [ + "eslint-config-egg/typescript", + "eslint-config-egg/lib/rules/enforce-node-prefix" + ] +} diff --git a/nodejs/.gitignore b/nodejs/.gitignore new file mode 100644 index 000000000..62a16b936 --- /dev/null +++ b/nodejs/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +*.d.ts +*.js +rocketmq-all-*/ +nohup.out +dist/ +coverage/ diff --git a/nodejs/LICENSE b/nodejs/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/nodejs/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/nodejs/README-CN.md b/nodejs/README-CN.md new file mode 100644 index 000000000..183bad4e4 --- /dev/null +++ b/nodejs/README-CN.md @@ -0,0 +1,101 @@ +# Apache RocketMQ Node.js 客户端 + +[English](README.md) | 简体中文 | [RocketMQ 官网](https://rocketmq.apache.org/) + +## 概述 + +在开始客户端的部分之前，所需的一些前期工作（或者参照[这里](https://rocketmq.apache.org/zh/docs/quickStart/01quickstart/)）： + +1. 准备 [Node.js](https://nodejs.dev/zh-cn/download/) 环境。Node.js 16.19.0 是确保客户端运行的最小版本，Node.js >= 18.17.0 是推荐版本； +2. 部署 namesrv，broker 以及 [proxy](https://github.com/apache/rocketmq/tree/develop/proxy) 组件。 + +## 快速开始 + +我们使用 npm 作为依赖管理和发布的工具。你可以在 npm 的[官方网站](https://npmjs.com/)了解到关于它的更多信息。 +这里是一些在开发阶段你会使用到的 npm 命令： + +```shell +# 自动安装工程相关的依赖 +npm install +# 初始化 grpc 代码 +npm run init +# 运行单元测试 +npm test +``` + +开启 grpc-js 的调试日志： + +```bash +GRPC_TRACE=compression GRPC_VERBOSITY=debug GRPC_TRACE=all npm test +``` + +## 发布步骤 + +执行以下命令： + +```shell +# 构建包并将包发布到远程 npm 仓库 +npm publish +``` + +## 示例 + +### 普通消息 + +发送消息 + +```ts +import { Producer } from 'rocketmq-client-nodejs'; + +const producer = new Producer({ + endpoints: '127.0.0.1:8081', +}); +await producer.startup(); + +const receipt = await producer.send({ + topic: 'TopicTest', + tag: 'nodejs-demo', + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄', + now: Date(), + })), +}); +console.log(receipt); +``` + +消费消息 + +```ts +import { SimpleConsumer } from 'rocketmq-client-nodejs'; + +const simpleConsumer = new SimpleConsumer({ + consumerGroup: 'nodejs-demo-group', + endpoints: '127.0.0.1:8081', + subscriptions: new Map().set('TopicTest', 'nodejs-demo'), +}); +await simpleConsumer.startup(); + +const messages = await simpleConsumer.receive(20); +console.log('got %d messages', messages.length); +for (const message of messages) { + console.log(message); + console.log('body=%o', message.body.toString()); + await simpleConsumer.ack(message); +} +``` + +## Current Progress + +### Message Type + +- [x] NORMAL +- [x] FIFO +- [x] DELAY +- [x] TRANSACTION + +### Client Type + +- [x] PRODUCER +- [x] SIMPLE_CONSUMER +- [ ] PULL_CONSUMER +- [ ] PUSH_CONSUMER diff --git a/nodejs/README.md b/nodejs/README.md new file mode 100644 index 000000000..992bc0a84 --- /dev/null +++ b/nodejs/README.md @@ -0,0 +1,100 @@ +# The Node.js Implementation of Apache RocketMQ Client + +English | [简体中文](README-CN.md) | [RocketMQ Website](https://rocketmq.apache.org/) + +## Overview + +Here are some preparations you may need to know (or refer to [here](https://rocketmq.apache.org/docs/quickStart/01quickstart/)). + +1. [Node.js](https://nodejs.dev/en/download/) 16.19.0 is the minimum version required, Node.js >= 18.17.0 is the recommended version. +2. Setup namesrv, broker, and [proxy](https://github.com/apache/rocketmq/tree/develop/proxy). + +## Getting Started + +We are using npm as the dependency management & publishing tool. You can find out more details about npm from its [website](https://npmjs.com/). Here is the related command of npm you may use for development. + +```shell +# Installs the project dependencies. +npm install +# Init grpc codes. +npm run init +# Run the unit tests. +npm test +``` + +Enable trace debug log for grpc-js: + +```bash +GRPC_TRACE=compression GRPC_VERBOSITY=debug GRPC_TRACE=all npm test +``` + +## Publishing Steps + +To publish a package to npm, please register an account in advance, then execute the following command. + +```shell +# Builds a package and publishes it to the npm repository. +npm publish +``` + +## Examples + +### Normal Message + +Producer + +```ts +import { Producer } from 'rocketmq-client-nodejs'; + +const producer = new Producer({ + endpoints: '127.0.0.1:8081', +}); +await producer.startup(); + +const receipt = await producer.send({ + topic: 'TopicTest', + tag: 'nodejs-demo', + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄', + now: Date(), + })), +}); +console.log(receipt); +``` + +SimpleConsumer + +```ts +import { SimpleConsumer } from 'rocketmq-client-nodejs'; + +const simpleConsumer = new SimpleConsumer({ + consumerGroup: 'nodejs-demo-group', + endpoints: '127.0.0.1:8081', + subscriptions: new Map().set('TopicTest', 'nodejs-demo'), +}); +await simpleConsumer.startup(); + +const messages = await simpleConsumer.receive(20); +console.log('got %d messages', messages.length); +for (const message of messages) { + console.log(message); + console.log('body=%o', message.body.toString()); + await simpleConsumer.ack(message); +} +``` + +## Current Progress + +### Message Type + +- [x] NORMAL +- [x] FIFO +- [x] DELAY +- [x] TRANSACTION + +### Client Type + +- [x] PRODUCER +- [x] SIMPLE_CONSUMER +- [ ] PULL_CONSUMER +- [ ] PUSH_CONSUMER diff --git a/nodejs/examples/Producer.cjs b/nodejs/examples/Producer.cjs new file mode 100644 index 000000000..4f0273c0a --- /dev/null +++ b/nodejs/examples/Producer.cjs @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-var-requires */ +const { Producer } = require('..'); + +async function main() { + const producer = new Producer({ + endpoints: '127.0.0.1:8081', + logger: console, + }); + await producer.startup(); + + const receipt = await producer.send({ + topic: 'TopicTest', + tag: 'nodejs-demo', + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄', + now: Date(), + })), + }); + console.log(receipt); + + await producer.shutdown(); + // process.exit(0); +} + +main(); diff --git a/nodejs/examples/Producer.ts b/nodejs/examples/Producer.ts new file mode 100644 index 000000000..c90c761f0 --- /dev/null +++ b/nodejs/examples/Producer.ts @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Producer } from '..'; + +const producer = new Producer({ + endpoints: '127.0.0.1:8081', +}); +await producer.startup(); + +const receipt = await producer.send({ + topic: 'TopicTest', + tag: 'nodejs-demo', + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄', + now: Date(), + })), +}); +console.log(receipt); diff --git a/nodejs/examples/SimpleConsumer.cjs b/nodejs/examples/SimpleConsumer.cjs new file mode 100644 index 000000000..babbbb392 --- /dev/null +++ b/nodejs/examples/SimpleConsumer.cjs @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-var-requires */ +const { SimpleConsumer } = require('..'); + +async function main() { + const simpleConsumer = new SimpleConsumer({ + consumerGroup: 'nodejs-demo-group', + endpoints: '127.0.0.1:8081', + subscriptions: new Map().set('TopicTest', 'nodejs-demo'), + }); + await simpleConsumer.startup(); + + const messages = await simpleConsumer.receive(20); + console.log('got %d messages', messages.length); + for (const message of messages) { + console.log(message); + console.log('body=%o', message.body.toString()); + await simpleConsumer.ack(message); + } + await simpleConsumer.shutdown(); +} + +main(); diff --git a/nodejs/examples/SimpleConsumer.ts b/nodejs/examples/SimpleConsumer.ts new file mode 100644 index 000000000..c58d9d170 --- /dev/null +++ b/nodejs/examples/SimpleConsumer.ts @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SimpleConsumer } from '..'; + +const simpleConsumer = new SimpleConsumer({ + consumerGroup: 'nodejs-demo-group', + endpoints: '127.0.0.1:8081', + subscriptions: new Map().set('TopicTest', 'nodejs-demo'), +}); +await simpleConsumer.startup(); + +const messages = await simpleConsumer.receive(20); +console.log('got %d messages', messages.length); +for (const message of messages) { + console.log(message); + console.log('body=%o', message.body.toString()); + await simpleConsumer.ack(message); +} diff --git a/nodejs/package.json b/nodejs/package.json new file mode 100644 index 000000000..45978ec80 --- /dev/null +++ b/nodejs/package.json @@ -0,0 +1,61 @@ +{ + "name": "rocketmq-client-nodejs", + "version": "1.0.0", + "description": "RocketMQ Node.js Client", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "proto", + "dist", + "src" + ], + "scripts": { + "init": "npm run build:grpc", + "test": "egg-bin test", + "cov": "egg-bin cov", + "lint": "eslint src test", + "build:grpc": "scripts/build-grpc.sh", + "clean": "rm -rf dist *.d.ts", + "tsc": "tsc -p tsconfig.prod.json", + "build": "npm run clean && npm run build:grpc && npm run tsc", + "start-rocketmq": "sh test/start-rocketmq.sh", + "ci": "npm run lint && npm run build && npm run cov", + "prepublishOnly": "npm run build" + }, + "repository": { + "type": "git", + "url": "git+ssh://git@github.com/apache/rocketmq-clients.git", + "directory": "nodejs" + }, + "keywords": [ + "rocketmq" + ], + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/apache/rocketmq-clients/issues" + }, + "homepage": "https://github.com/apache/rocketmq-clients/blob/master/nodejs/README.md", + "devDependencies": { + "@eggjs/tsconfig": "^1.3.3", + "@types/google-protobuf": "^3.15.6", + "@types/mocha": "^10.0.1", + "@types/node": "^20.5.7", + "egg-bin": "^6.4.2", + "eslint": "^8.48.0", + "eslint-config-egg": "^12.2.1", + "grpc-tools": "^1.12.4", + "grpc_tools_node_protoc_ts": "^5.3.3", + "typescript": "^5.2.2" + }, + "dependencies": { + "@grpc/grpc-js": "^1.9.1", + "@node-rs/crc32": "^1.7.2", + "address": "^1.2.2", + "egg-logger": "^3.4.0", + "google-protobuf": "^3.21.2", + "siphash24": "^1.3.1" + }, + "engines": { + "node": ">=16.19.0" + } +} diff --git a/nodejs/proto/apache/rocketmq/v2/admin.proto b/nodejs/proto/apache/rocketmq/v2/admin.proto new file mode 100644 index 000000000..7dbb7027d --- /dev/null +++ b/nodejs/proto/apache/rocketmq/v2/admin.proto @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package apache.rocketmq.v2; + +option cc_enable_arenas = true; +option csharp_namespace = "Apache.Rocketmq.V2"; +option java_multiple_files = true; +option java_package = "apache.rocketmq.v2"; +option java_generate_equals_and_hash = true; +option java_string_check_utf8 = true; +option java_outer_classname = "MQAdmin"; + +message ChangeLogLevelRequest { + enum Level { + TRACE = 0; + DEBUG = 1; + INFO = 2; + WARN = 3; + ERROR = 4; + } + Level level = 1; +} + +message ChangeLogLevelResponse { string remark = 1; } + +service Admin { + rpc ChangeLogLevel(ChangeLogLevelRequest) returns (ChangeLogLevelResponse) {} +} \ No newline at end of file diff --git a/nodejs/proto/apache/rocketmq/v2/definition.proto b/nodejs/proto/apache/rocketmq/v2/definition.proto new file mode 100644 index 000000000..753bfcebe --- /dev/null +++ b/nodejs/proto/apache/rocketmq/v2/definition.proto @@ -0,0 +1,564 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; + +package apache.rocketmq.v2; + +option csharp_namespace = "Apache.Rocketmq.V2"; +option java_multiple_files = true; +option java_package = "apache.rocketmq.v2"; +option java_generate_equals_and_hash = true; +option java_string_check_utf8 = true; +option java_outer_classname = "MQDomain"; + +enum TransactionResolution { + TRANSACTION_RESOLUTION_UNSPECIFIED = 0; + COMMIT = 1; + ROLLBACK = 2; +} + +enum TransactionSource { + SOURCE_UNSPECIFIED = 0; + SOURCE_CLIENT = 1; + SOURCE_SERVER_CHECK = 2; +} + +enum Permission { + PERMISSION_UNSPECIFIED = 0; + NONE = 1; + READ = 2; + WRITE = 3; + READ_WRITE = 4; +} + +enum FilterType { + FILTER_TYPE_UNSPECIFIED = 0; + TAG = 1; + SQL = 2; +} + +message FilterExpression { + FilterType type = 1; + string expression = 2; +} + +message RetryPolicy { + int32 max_attempts = 1; + oneof strategy { + ExponentialBackoff exponential_backoff = 2; + CustomizedBackoff customized_backoff = 3; + } +} + +// https://en.wikipedia.org/wiki/Exponential_backoff +message ExponentialBackoff { + google.protobuf.Duration initial = 1; + google.protobuf.Duration max = 2; + float multiplier = 3; +} + +message CustomizedBackoff { + // To support classic backoff strategy which is arbitrary defined by end users. + // Typical values are: `1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h` + repeated google.protobuf.Duration next = 1; +} + +message Resource { + string resource_namespace = 1; + + // Resource name identifier, which remains unique within the abstract resource + // namespace. + string name = 2; +} + +message SubscriptionEntry { + Resource topic = 1; + FilterExpression expression = 2; +} + +enum AddressScheme { + ADDRESS_SCHEME_UNSPECIFIED = 0; + IPv4 = 1; + IPv6 = 2; + DOMAIN_NAME = 3; +} + +message Address { + string host = 1; + int32 port = 2; +} + +message Endpoints { + AddressScheme scheme = 1; + repeated Address addresses = 2; +} + +message Broker { + // Name of the broker + string name = 1; + + // Broker index. Canonically, index = 0 implies that the broker is playing + // leader role while brokers with index > 0 play follower role. + int32 id = 2; + + // Address of the broker, complying with the following scheme + // 1. dns:[//authority/]host[:port] + // 2. ipv4:address[:port][,address[:port],...] – IPv4 addresses + // 3. ipv6:address[:port][,address[:port],...] – IPv6 addresses + Endpoints endpoints = 3; +} + +message MessageQueue { + Resource topic = 1; + int32 id = 2; + Permission permission = 3; + Broker broker = 4; + repeated MessageType accept_message_types = 5; +} + +enum MessageType { + MESSAGE_TYPE_UNSPECIFIED = 0; + + NORMAL = 1; + + // Sequenced message + FIFO = 2; + + // Messages that are delivered after the specified duration. + DELAY = 3; + + // Messages that are transactional. Only committed messages are delivered to + // subscribers. + TRANSACTION = 4; +} + +enum DigestType { + DIGEST_TYPE_UNSPECIFIED = 0; + + // CRC algorithm achieves goal of detecting random data error with lowest + // computation overhead. + CRC32 = 1; + + // MD5 algorithm achieves good balance between collision rate and computation + // overhead. + MD5 = 2; + + // SHA-family has substantially fewer collision with fair amount of + // computation. + SHA1 = 3; +} + +// When publishing messages to or subscribing messages from brokers, clients +// shall include or validate digests of message body to ensure data integrity. +// +// For message publishing, when an invalid digest were detected, brokers need +// respond client with BAD_REQUEST. +// +// For messages subscription, when an invalid digest were detected, consumers +// need to handle this case according to message type: +// 1) Standard messages should be negatively acknowledged instantly, causing +// immediate re-delivery; 2) FIFO messages require special RPC, to re-fetch +// previously acquired messages batch; +message Digest { + DigestType type = 1; + string checksum = 2; +} + +enum ClientType { + CLIENT_TYPE_UNSPECIFIED = 0; + PRODUCER = 1; + PUSH_CONSUMER = 2; + SIMPLE_CONSUMER = 3; + PULL_CONSUMER = 4; +} + +enum Encoding { + ENCODING_UNSPECIFIED = 0; + + IDENTITY = 1; + + GZIP = 2; +} + +message SystemProperties { + // Tag, which is optional. + optional string tag = 1; + + // Message keys + repeated string keys = 2; + + // Message identifier, client-side generated, remains unique. + // if message_id is empty, the send message request will be aborted with + // status `INVALID_ARGUMENT` + string message_id = 3; + + // Message body digest + Digest body_digest = 4; + + // Message body encoding. Candidate options are identity, gzip, snappy etc. + Encoding body_encoding = 5; + + // Message type, normal, FIFO or transactional. + MessageType message_type = 6; + + // Message born time-point. + google.protobuf.Timestamp born_timestamp = 7; + + // Message born host. Valid options are IPv4, IPv6 or client host domain name. + string born_host = 8; + + // Time-point at which the message is stored in the broker, which is absent + // for message publishing. + optional google.protobuf.Timestamp store_timestamp = 9; + + // The broker that stores this message. It may be broker name, IP or arbitrary + // identifier that uniquely identify the server. + string store_host = 10; + + // Time-point at which broker delivers to clients, which is optional. + optional google.protobuf.Timestamp delivery_timestamp = 11; + + // If a message is acquired by way of POP, this field holds the receipt, + // which is absent for message publishing. + // Clients use the receipt to acknowledge or negatively acknowledge the + // message. + optional string receipt_handle = 12; + + // Message queue identifier in which a message is physically stored. + int32 queue_id = 13; + + // Message-queue offset at which a message is stored, which is absent for + // message publishing. + optional int64 queue_offset = 14; + + // Period of time servers would remain invisible once a message is acquired. + optional google.protobuf.Duration invisible_duration = 15; + + // Business code may failed to process messages for the moment. Hence, clients + // may request servers to deliver them again using certain back-off strategy, + // the attempt is 1 not 0 if message is delivered first time, and it is absent + // for message publishing. + optional int32 delivery_attempt = 16; + + // Define the group name of message in the same topic, which is optional. + optional string message_group = 17; + + // Trace context for each message, which is optional. + optional string trace_context = 18; + + // If a transactional message stay unresolved for more than + // `transaction_orphan_threshold`, it would be regarded as an + // orphan. Servers that manages orphan messages would pick up + // a capable publisher to resolve + optional google.protobuf.Duration orphaned_transaction_recovery_duration = 19; + + // Information to identify whether this message is from dead letter queue. + optional DeadLetterQueue dead_letter_queue = 20; +} + +message DeadLetterQueue { + // Original topic for this DLQ message. + string topic = 1; + // Original message id for this DLQ message. + string message_id = 2; +} + +message Message { + + Resource topic = 1; + + // User defined key-value pairs. + // If user_properties contain the reserved keys by RocketMQ, + // the send message request will be aborted with status `INVALID_ARGUMENT`. + // See below links for the reserved keys + // https://github.com/apache/rocketmq/blob/master/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java#L58 + map user_properties = 2; + + SystemProperties system_properties = 3; + + bytes body = 4; +} + +message Assignment { + MessageQueue message_queue = 1; +} + +enum Code { + CODE_UNSPECIFIED = 0; + + // Generic code for success. + OK = 20000; + + // Generic code for multiple return results. + MULTIPLE_RESULTS = 30000; + + // Generic code for bad request, indicating that required fields or headers are missing. + BAD_REQUEST = 40000; + // Format of access point is illegal. + ILLEGAL_ACCESS_POINT = 40001; + // Format of topic is illegal. + ILLEGAL_TOPIC = 40002; + // Format of consumer group is illegal. + ILLEGAL_CONSUMER_GROUP = 40003; + // Format of message tag is illegal. + ILLEGAL_MESSAGE_TAG = 40004; + // Format of message key is illegal. + ILLEGAL_MESSAGE_KEY = 40005; + // Format of message group is illegal. + ILLEGAL_MESSAGE_GROUP = 40006; + // Format of message property key is illegal. + ILLEGAL_MESSAGE_PROPERTY_KEY = 40007; + // Transaction id is invalid. + INVALID_TRANSACTION_ID = 40008; + // Format of message id is illegal. + ILLEGAL_MESSAGE_ID = 40009; + // Format of filter expression is illegal. + ILLEGAL_FILTER_EXPRESSION = 40010; + // The invisible time of request is invalid. + ILLEGAL_INVISIBLE_TIME = 40011; + // The delivery timestamp of message is invalid. + ILLEGAL_DELIVERY_TIME = 40012; + // Receipt handle of message is invalid. + INVALID_RECEIPT_HANDLE = 40013; + // Message property conflicts with its type. + MESSAGE_PROPERTY_CONFLICT_WITH_TYPE = 40014; + // Client type could not be recognized. + UNRECOGNIZED_CLIENT_TYPE = 40015; + // Message is corrupted. + MESSAGE_CORRUPTED = 40016; + // Request is rejected due to missing of x-mq-client-id header. + CLIENT_ID_REQUIRED = 40017; + // Polling time is illegal. + ILLEGAL_POLLING_TIME = 40018; + + // Generic code indicates that the client request lacks valid authentication + // credentials for the requested resource. + UNAUTHORIZED = 40100; + + // Generic code indicates that the account is suspended due to overdue of payment. + PAYMENT_REQUIRED = 40200; + + // Generic code for the case that user does not have the permission to operate. + FORBIDDEN = 40300; + + // Generic code for resource not found. + NOT_FOUND = 40400; + // Message not found from server. + MESSAGE_NOT_FOUND = 40401; + // Topic resource does not exist. + TOPIC_NOT_FOUND = 40402; + // Consumer group resource does not exist. + CONSUMER_GROUP_NOT_FOUND = 40403; + + // Generic code representing client side timeout when connecting to, reading data from, or write data to server. + REQUEST_TIMEOUT = 40800; + + // Generic code represents that the request entity is larger than limits defined by server. + PAYLOAD_TOO_LARGE = 41300; + // Message body size exceeds the threshold. + MESSAGE_BODY_TOO_LARGE = 41301; + + // Generic code for use cases where pre-conditions are not met. + // For example, if a producer instance is used to publish messages without prior start() invocation, + // this error code will be raised. + PRECONDITION_FAILED = 42800; + + // Generic code indicates that too many requests are made in short period of duration. + // Requests are throttled. + TOO_MANY_REQUESTS = 42900; + + // Generic code for the case that the server is unwilling to process the request because its header fields are too large. + // The request may be resubmitted after reducing the size of the request header fields. + REQUEST_HEADER_FIELDS_TOO_LARGE = 43100; + // Message properties total size exceeds the threshold. + MESSAGE_PROPERTIES_TOO_LARGE = 43101; + + // Generic code indicates that server/client encountered an unexpected + // condition that prevented it from fulfilling the request. + INTERNAL_ERROR = 50000; + // Code indicates that the server encountered an unexpected condition + // that prevented it from fulfilling the request. + // This error response is a generic "catch-all" response. + // Usually, this indicates the server cannot find a better alternative + // error code to response. Sometimes, server administrators log error + // responses like the 500 status code with more details about the request + // to prevent the error from happening again in the future. + // + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/500 + INTERNAL_SERVER_ERROR = 50001; + // The HA-mechanism is not working now. + HA_NOT_AVAILABLE = 50002; + + // Generic code means that the server or client does not support the + // functionality required to fulfill the request. + NOT_IMPLEMENTED = 50100; + + // Generic code represents that the server, which acts as a gateway or proxy, + // does not get an satisfied response in time from its upstream servers. + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/504 + PROXY_TIMEOUT = 50400; + // Message persistence timeout. + MASTER_PERSISTENCE_TIMEOUT = 50401; + // Slave persistence timeout. + SLAVE_PERSISTENCE_TIMEOUT = 50402; + + // Generic code for unsupported operation. + UNSUPPORTED = 50500; + // Operation is not allowed in current version. + VERSION_UNSUPPORTED = 50501; + // Not allowed to verify message. Chances are that you are verifying + // a FIFO message, as is violating FIFO semantics. + VERIFY_FIFO_MESSAGE_UNSUPPORTED = 50502; + + // Generic code for failed message consumption. + FAILED_TO_CONSUME_MESSAGE = 60000; +} + +message Status { + Code code = 1; + string message = 2; +} + +enum Language { + LANGUAGE_UNSPECIFIED = 0; + JAVA = 1; + CPP = 2; + DOT_NET = 3; + GOLANG = 4; + RUST = 5; + PYTHON = 6; + PHP = 7; + NODE_JS = 8; + RUBY = 9; + OBJECTIVE_C = 10; + DART = 11; + KOTLIN = 12; +} + +// User Agent +message UA { + // SDK language + Language language = 1; + + // SDK version + string version = 2; + + // Platform details, including OS name, version, arch etc. + string platform = 3; + + // Hostname of the node + string hostname = 4; +} + +message Settings { + // Configurations for all clients. + optional ClientType client_type = 1; + + optional Endpoints access_point = 2; + + // If publishing of messages encounters throttling or server internal errors, + // publishers should implement automatic retries after progressive longer + // back-offs for consecutive errors. + // + // When processing message fails, `backoff_policy` describes an interval + // after which the message should be available to consume again. + // + // For FIFO messages, the interval should be relatively small because + // messages of the same message group would not be readily available until + // the prior one depletes its lifecycle. + optional RetryPolicy backoff_policy = 3; + + // Request timeout for RPCs excluding long-polling. + optional google.protobuf.Duration request_timeout = 4; + + oneof pub_sub { + Publishing publishing = 5; + + Subscription subscription = 6; + } + + // User agent details + UA user_agent = 7; + + Metric metric = 8; +} + +message Publishing { + // Publishing settings below here is appointed by client, thus it is + // unnecessary for server to push at present. + // + // List of topics to which messages will publish to. + repeated Resource topics = 1; + + // If the message body size exceeds `max_body_size`, broker servers would + // reject the request. As a result, it is advisable that Producer performs + // client-side check validation. + int32 max_body_size = 2; + + // When `validate_message_type` flag set `false`, no need to validate message's type + // with messageQueue's `accept_message_types` before publishing. + bool validate_message_type = 3; +} + +message Subscription { + // Subscription settings below here is appointed by client, thus it is + // unnecessary for server to push at present. + // + // Consumer group. + optional Resource group = 1; + + // Subscription for consumer. + repeated SubscriptionEntry subscriptions = 2; + + // Subscription settings below here are from server, it is essential for + // server to push. + // + // When FIFO flag is `true`, messages of the same message group are processed + // in first-in-first-out manner. + // + // Brokers will not deliver further messages of the same group until prior + // ones are completely acknowledged. + optional bool fifo = 3; + + // Message receive batch size here is essential for push consumer. + optional int32 receive_batch_size = 4; + + // Long-polling timeout for `ReceiveMessageRequest`, which is essential for + // push consumer. + optional google.protobuf.Duration long_polling_timeout = 5; +} + +message Metric { + // Indicates that if client should export local metrics to server. + bool on = 1; + + // The endpoint that client metrics should be exported to, which is required if the switch is on. + optional Endpoints endpoints = 2; +} + +enum QueryOffsetPolicy { + // Use this option if client wishes to playback all existing messages. + BEGINNING = 0; + + // Use this option if client wishes to skip all existing messages. + END = 1; + + // Use this option if time-based seek is targeted. + TIMESTAMP = 2; +} \ No newline at end of file diff --git a/nodejs/proto/apache/rocketmq/v2/service.proto b/nodejs/proto/apache/rocketmq/v2/service.proto new file mode 100644 index 000000000..f662f769e --- /dev/null +++ b/nodejs/proto/apache/rocketmq/v2/service.proto @@ -0,0 +1,411 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +import "apache/rocketmq/v2/definition.proto"; + +package apache.rocketmq.v2; + +option csharp_namespace = "Apache.Rocketmq.V2"; +option java_multiple_files = true; +option java_package = "apache.rocketmq.v2"; +option java_generate_equals_and_hash = true; +option java_string_check_utf8 = true; +option java_outer_classname = "MQService"; + +// Topics are destination of messages to publish to or subscribe from. Similar +// to domain names, they will be addressable after resolution through the +// provided access point. +// +// Access points are usually the addresses of name servers, which fulfill +// service discovery, load-balancing and other auxiliary services. Name servers +// receive periodic heartbeats from affiliate brokers and erase those which +// failed to maintain alive status. +// +// Name servers answer queries of QueryRouteRequest, responding clients with +// addressable message-queues, which they may directly publish messages to or +// subscribe messages from. +// +// QueryRouteRequest shall include source endpoints, aka, configured +// access-point, which annotates tenant-id, instance-id or other +// vendor-specific settings. Purpose-built name servers may respond customized +// results based on these particular requirements. +message QueryRouteRequest { + Resource topic = 1; + Endpoints endpoints = 2; +} + +message QueryRouteResponse { + Status status = 1; + + repeated MessageQueue message_queues = 2; +} + +message SendMessageRequest { + repeated Message messages = 1; +} + +message SendResultEntry { + Status status = 1; + string message_id = 2; + string transaction_id = 3; + int64 offset = 4; +} + +message SendMessageResponse { + Status status = 1; + + // Some implementation may have partial failure issues. Client SDK developers are expected to inspect + // each entry for best certainty. + repeated SendResultEntry entries = 2; +} + +message QueryAssignmentRequest { + Resource topic = 1; + Resource group = 2; + Endpoints endpoints = 3; +} + +message QueryAssignmentResponse { + Status status = 1; + repeated Assignment assignments = 2; +} + +message ReceiveMessageRequest { + Resource group = 1; + MessageQueue message_queue = 2; + FilterExpression filter_expression = 3; + int32 batch_size = 4; + // Required if client type is simple consumer. + optional google.protobuf.Duration invisible_duration = 5; + // For message auto renew and clean + bool auto_renew = 6; + optional google.protobuf.Duration long_polling_timeout = 7; +} + +message ReceiveMessageResponse { + oneof content { + Status status = 1; + Message message = 2; + // The timestamp that brokers start to deliver status line or message. + google.protobuf.Timestamp delivery_timestamp = 3; + } +} + +message AckMessageEntry { + string message_id = 1; + string receipt_handle = 2; +} + +message AckMessageRequest { + Resource group = 1; + Resource topic = 2; + repeated AckMessageEntry entries = 3; +} + +message AckMessageResultEntry { + string message_id = 1; + string receipt_handle = 2; + + // Acknowledge result may be acquired through inspecting + // `status.code`; In case acknowledgement failed, `status.message` + // is the explanation of the failure. + Status status = 3; +} + +message AckMessageResponse { + + // RPC tier status, which is used to represent RPC-level errors including + // authentication, authorization, throttling and other general failures. + Status status = 1; + + repeated AckMessageResultEntry entries = 2; +} + +message ForwardMessageToDeadLetterQueueRequest { + Resource group = 1; + Resource topic = 2; + string receipt_handle = 3; + string message_id = 4; + int32 delivery_attempt = 5; + int32 max_delivery_attempts = 6; +} + +message ForwardMessageToDeadLetterQueueResponse { Status status = 1; } + +message HeartbeatRequest { + optional Resource group = 1; + ClientType client_type = 2; +} + +message HeartbeatResponse { Status status = 1; } + +message EndTransactionRequest { + Resource topic = 1; + string message_id = 2; + string transaction_id = 3; + TransactionResolution resolution = 4; + TransactionSource source = 5; + string trace_context = 6; +} + +message EndTransactionResponse { Status status = 1; } + +message PrintThreadStackTraceCommand { string nonce = 1; } + +message ThreadStackTrace { + string nonce = 1; + optional string thread_stack_trace = 2; +} + +message VerifyMessageCommand { + string nonce = 1; + Message message = 2; +} + +message VerifyMessageResult { + string nonce = 1; +} + +message RecoverOrphanedTransactionCommand { + Message message = 1; + string transaction_id = 2; +} + +message TelemetryCommand { + optional Status status = 1; + + oneof command { + // Client settings + Settings settings = 2; + + // These messages are from client. + // + // Report thread stack trace to server. + ThreadStackTrace thread_stack_trace = 3; + + // Report message verify result to server. + VerifyMessageResult verify_message_result = 4; + + // There messages are from server. + // + // Request client to recover the orphaned transaction message. + RecoverOrphanedTransactionCommand recover_orphaned_transaction_command = 5; + + // Request client to print thread stack trace. + PrintThreadStackTraceCommand print_thread_stack_trace_command = 6; + + // Request client to verify the consumption of the appointed message. + VerifyMessageCommand verify_message_command = 7; + } +} + +message NotifyClientTerminationRequest { + // Consumer group, which is absent for producer. + optional Resource group = 1; +} + +message NotifyClientTerminationResponse { Status status = 1; } + +message ChangeInvisibleDurationRequest { + Resource group = 1; + Resource topic = 2; + + // Unique receipt handle to identify message to change + string receipt_handle = 3; + + // New invisible duration + google.protobuf.Duration invisible_duration = 4; + + // For message tracing + string message_id = 5; +} + +message ChangeInvisibleDurationResponse { + Status status = 1; + + // Server may generate a new receipt handle for the message. + string receipt_handle = 2; +} + +message PullMessageRequest { + Resource group = 1; + MessageQueue message_queue = 2; + int64 offset = 3; + int32 batch_size = 4; + FilterExpression filter_expression = 5; + google.protobuf.Duration long_polling_timeout = 6; +} + +message PullMessageResponse { + oneof content { + Status status = 1; + Message message = 2; + int64 next_offset = 3; + } +} + +message UpdateOffsetRequest { + Resource group = 1; + MessageQueue message_queue = 2; + int64 offset = 3; +} + +message UpdateOffsetResponse { + Status status = 1; +} + +message GetOffsetRequest { + Resource group = 1; + MessageQueue message_queue = 2; +} + +message GetOffsetResponse { + Status status = 1; + int64 offset = 2; +} + +message QueryOffsetRequest { + MessageQueue message_queue = 1; + QueryOffsetPolicy query_offset_policy = 2; + optional google.protobuf.Timestamp timestamp = 3; +} + +message QueryOffsetResponse { + Status status = 1; + int64 offset = 2; +} + +// For all the RPCs in MessagingService, the following error handling policies +// apply: +// +// If the request doesn't bear a valid authentication credential, return a +// response with common.status.code == `UNAUTHENTICATED`. If the authenticated +// user is not granted with sufficient permission to execute the requested +// operation, return a response with common.status.code == `PERMISSION_DENIED`. +// If the per-user-resource-based quota is exhausted, return a response with +// common.status.code == `RESOURCE_EXHAUSTED`. If any unexpected server-side +// errors raise, return a response with common.status.code == `INTERNAL`. +service MessagingService { + + // Queries the route entries of the requested topic in the perspective of the + // given endpoints. On success, servers should return a collection of + // addressable message-queues. Note servers may return customized route + // entries based on endpoints provided. + // + // If the requested topic doesn't exist, returns `NOT_FOUND`. + // If the specific endpoints is empty, returns `INVALID_ARGUMENT`. + rpc QueryRoute(QueryRouteRequest) returns (QueryRouteResponse) {} + + // Producer or consumer sends HeartbeatRequest to servers periodically to + // keep-alive. Additionally, it also reports client-side configuration, + // including topic subscription, load-balancing group name, etc. + // + // Returns `OK` if success. + // + // If a client specifies a language that is not yet supported by servers, + // returns `INVALID_ARGUMENT` + rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse) {} + + // Delivers messages to brokers. + // Clients may further: + // 1. Refine a message destination to message-queues which fulfills parts of + // FIFO semantic; + // 2. Flag a message as transactional, which keeps it invisible to consumers + // until it commits; + // 3. Time a message, making it invisible to consumers till specified + // time-point; + // 4. And more... + // + // Returns message-id or transaction-id with status `OK` on success. + // + // If the destination topic doesn't exist, returns `NOT_FOUND`. + rpc SendMessage(SendMessageRequest) returns (SendMessageResponse) {} + + // Queries the assigned route info of a topic for current consumer, + // the returned assignment result is decided by server-side load balancer. + // + // If the corresponding topic doesn't exist, returns `NOT_FOUND`. + // If the specific endpoints is empty, returns `INVALID_ARGUMENT`. + rpc QueryAssignment(QueryAssignmentRequest) returns (QueryAssignmentResponse) { + } + + // Receives messages from the server in batch manner, returns a set of + // messages if success. The received messages should be acked or redelivered + // after processed. + // + // If the pending concurrent receive requests exceed the quota of the given + // consumer group, returns `UNAVAILABLE`. If the upstream store server hangs, + // return `DEADLINE_EXCEEDED` in a timely manner. If the corresponding topic + // or consumer group doesn't exist, returns `NOT_FOUND`. If there is no new + // message in the specific topic, returns `OK` with an empty message set. + // Please note that client may suffer from false empty responses. + // + // If failed to receive message from remote, server must return only one + // `ReceiveMessageResponse` as the reply to the request, whose `Status` indicates + // the specific reason of failure, otherwise, the reply is considered successful. + rpc ReceiveMessage(ReceiveMessageRequest) returns (stream ReceiveMessageResponse) { + } + + // Acknowledges the message associated with the `receipt_handle` or `offset` + // in the `AckMessageRequest`, it means the message has been successfully + // processed. Returns `OK` if the message server remove the relevant message + // successfully. + // + // If the given receipt_handle is illegal or out of date, returns + // `INVALID_ARGUMENT`. + rpc AckMessage(AckMessageRequest) returns (AckMessageResponse) {} + + // Forwards one message to dead letter queue if the max delivery attempts is + // exceeded by this message at client-side, return `OK` if success. + rpc ForwardMessageToDeadLetterQueue(ForwardMessageToDeadLetterQueueRequest) + returns (ForwardMessageToDeadLetterQueueResponse) {} + + rpc PullMessage(PullMessageRequest) returns (stream PullMessageResponse) {} + + rpc UpdateOffset(UpdateOffsetRequest) returns (UpdateOffsetResponse) {} + + rpc GetOffset(GetOffsetRequest) returns (GetOffsetResponse) {} + + rpc QueryOffset(QueryOffsetRequest) returns (QueryOffsetResponse) {} + + // Commits or rollback one transactional message. + rpc EndTransaction(EndTransactionRequest) returns (EndTransactionResponse) {} + + // Once a client starts, it would immediately establishes bi-lateral stream + // RPCs with brokers, reporting its settings as the initiative command. + // + // When servers have need of inspecting client status, they would issue + // telemetry commands to clients. After executing received instructions, + // clients shall report command execution results through client-side streams. + rpc Telemetry(stream TelemetryCommand) returns (stream TelemetryCommand) {} + + // Notify the server that the client is terminated. + rpc NotifyClientTermination(NotifyClientTerminationRequest) returns (NotifyClientTerminationResponse) { + } + + // Once a message is retrieved from consume queue on behalf of the group, it + // will be kept invisible to other clients of the same group for a period of + // time. The message is supposed to be processed within the invisible + // duration. If the client, which is in charge of the invisible message, is + // not capable of processing the message timely, it may use + // ChangeInvisibleDuration to lengthen invisible duration. + rpc ChangeInvisibleDuration(ChangeInvisibleDurationRequest) returns (ChangeInvisibleDurationResponse) { + } +} \ No newline at end of file diff --git a/nodejs/scripts/build-grpc.sh b/nodejs/scripts/build-grpc.sh new file mode 100755 index 000000000..c68d6dd7c --- /dev/null +++ b/nodejs/scripts/build-grpc.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OS=$(echo `uname`|tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m) + +# Proto buf generation +PWD=$(pwd) +PATH_ROOT=$(dirname "$PWD") +PATH_PROTO_ROOT="${PATH_ROOT}/protos" +PATH_PROTO_OUTPUT="${PWD}/proto" +PATH_PROTO_OUTPUT_ROCKETMQ_V2="${PWD}/proto/apache/rocketmq/v2" + +PROTO_FILES=( + "apache/rocketmq/v2/admin.proto" + "apache/rocketmq/v2/definition.proto" + "apache/rocketmq/v2/service.proto" +) + +generateGrpc() { + PATH_PROTO=$1 + PATH_FILE=$2 + + echo "[protoc] Generating RPC for $PATH_PROTO/$PATH_FILE" + + # Tools to be installed by npm (see package.json) + # npm install grpc-tools --save-dev + # npm install grpc_tools_node_protoc_ts --save-dev + PROTOC_GEN_TS_PATH="${PWD}/node_modules/.bin/protoc-gen-ts" + PROTOC_GEN_GRPC_PATH="${PWD}/node_modules/.bin/grpc_tools_node_protoc_plugin" + + # commonjs + grpc_tools_node_protoc \ + --proto_path="${PATH_PROTO}" \ + --plugin="protoc-gen-ts=${PROTOC_GEN_TS_PATH}" \ + --plugin=protoc-gen-grpc=${PROTOC_GEN_GRPC_PATH} \ + --js_out="import_style=commonjs,binary:$PATH_PROTO_OUTPUT" \ + --ts_out="grpc_js:$PATH_PROTO_OUTPUT" \ + --grpc_out="grpc_js:$PATH_PROTO_OUTPUT" \ + "$PATH_PROTO/$PATH_FILE" + cp "$PATH_PROTO/$PATH_FILE" "${PATH_PROTO_OUTPUT_ROCKETMQ_V2}/" +} + +echo "" +echo "Removing old Proto Files: ${PATH_PROTO_OUTPUT}" +rm -rf $PATH_PROTO_OUTPUT +mkdir -p $PATH_PROTO_OUTPUT + +echo "" +echo "Compiling gRPC files" + +for proto_file in ${PROTO_FILES[@]}; do + echo "generate ${proto_file}" + generateGrpc $PATH_PROTO_ROOT "${proto_file}" +done + +echo "" +echo "DONE" diff --git a/nodejs/src/client/BaseClient.ts b/nodejs/src/client/BaseClient.ts new file mode 100644 index 000000000..ad67e0856 --- /dev/null +++ b/nodejs/src/client/BaseClient.ts @@ -0,0 +1,384 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { debuglog } from 'node:util'; +import { randomUUID } from 'node:crypto'; +import { Metadata } from '@grpc/grpc-js'; +import { + Settings as SettingsPB, + Status, + ClientType, + Code, +} from '../../proto/apache/rocketmq/v2/definition_pb'; +import { + QueryRouteRequest, + RecoverOrphanedTransactionCommand, + VerifyMessageCommand, + PrintThreadStackTraceCommand, + TelemetryCommand, + ThreadStackTrace, + HeartbeatRequest, + NotifyClientTerminationRequest, +} from '../../proto/apache/rocketmq/v2/service_pb'; +import { createResource, getRequestDateTime, sign } from '../util'; +import { TopicRouteData, Endpoints } from '../route'; +import { ClientException, StatusChecker } from '../exception'; +import { Settings } from './Settings'; +import { UserAgent } from './UserAgent'; +import { ILogger, getDefaultLogger } from './Logger'; +import { SessionCredentials } from './SessionCredentials'; +import { RpcClientManager } from './RpcClientManager'; +import { TelemetrySession } from './TelemetrySession'; +import { ClientId } from './ClientId'; + +const debug = debuglog('rocketmq-client-nodejs:client:BaseClient'); + +export interface BaseClientOptions { + sslEnabled?: boolean; + /** + * rocketmq cluster endpoints, e.g.: + * - 127.0.0.1:8081;127.0.0.1:8082 + * - 127.0.0.1:8081 + * - example.com + * - example.com:8443 + */ + endpoints: string; + sessionCredentials?: SessionCredentials; + requestTimeout?: number; + logger?: ILogger; + topics?: string[]; +} + +/** + * RocketMQ Base Client, Consumer and Producer should extends this class + * + * it handle: + * - RpcClient lifecycle, e.g: cleanup the idle clients + * - startup flow + * - periodic Task + */ +export abstract class BaseClient { + readonly clientId = ClientId.create(); + readonly clientType = ClientType.CLIENT_TYPE_UNSPECIFIED; + readonly sslEnabled: boolean; + readonly #sessionCredentials?: SessionCredentials; + protected readonly endpoints: Endpoints; + protected readonly isolated = new Map(); + protected readonly requestTimeout: number; + protected readonly topics = new Set(); + protected readonly topicRouteCache = new Map(); + protected readonly logger: ILogger; + protected readonly rpcClientManager: RpcClientManager; + readonly #telemetrySessions = new Map(); + #startupResolve?: () => void; + #startupReject?: (err: Error) => void; + #timers: NodeJS.Timeout[] = []; + + constructor(options: BaseClientOptions) { + this.logger = options.logger ?? getDefaultLogger(); + this.sslEnabled = options.sslEnabled === true; + this.endpoints = new Endpoints(options.endpoints); + this.#sessionCredentials = options.sessionCredentials; + // https://rocketmq.apache.org/docs/introduction/03limits/ + // Default request timeout is 3000ms + this.requestTimeout = options.requestTimeout ?? 3000; + this.rpcClientManager = new RpcClientManager(this, this.logger); + if (options.topics) { + for (const topic of options.topics) { + this.topics.add(topic); + } + } + } + + /** + * Startup flow + * https://github.com/apache/rocketmq-clients/blob/master/docs/workflow.md#startup + */ + async startup() { + this.logger.info('Begin to startup the rocketmq client, clientId=%s', this.clientId); + try { + await this.#startup(); + } catch (e) { + const err = new Error(`Startup the rocketmq client failed, clientId=${this.clientId}, error=${e}`); + this.logger.error(err); + err.cause = e; + throw err; + } + this.logger.info('Startup the rocketmq client successfully, clientId=%s', this.clientId); + } + + async #startup() { + // fetch topic route + await this.updateRoutes(); + // update topic route every 30s + this.#timers.push(setInterval(async () => { + this.updateRoutes(); + }, 30000)); + + // sync settings every 5m + this.#timers.push(setInterval(async () => { + this.#syncSettings(); + }, 5 * 60000)); + + // heartbeat every 10s + this.#timers.push(setInterval(async () => { + this.#doHeartbeat(); + }, 5 * 60000)); + + // doStats every 60s + // doStats() + + if (this.topics.size > 0) { + // wait for this first onSettingsCommand call + // eslint-disable-next-line @typescript-eslint/no-unused-vars + await new Promise((resolve, reject) => { + this.#startupReject = reject; + this.#startupResolve = resolve; + }); + this.#startupReject = undefined; + this.#startupResolve = undefined; + } + } + + async shutdown() { + this.logger.info('Begin to shutdown the rocketmq client, clientId=%s', this.clientId); + while (this.#timers.length > 0) { + const timer = this.#timers.pop(); + clearInterval(timer); + } + + await this.#notifyClientTermination(); + + this.logger.info('Begin to release all telemetry sessions, clientId=%s', this.clientId); + this.#releaseTelemetrySessions(); + this.logger.info('Release all telemetry sessions successfully, clientId=%s', this.clientId); + + this.rpcClientManager.close(); + this.logger.info('Shutdown the rocketmq client successfully, clientId=%s', this.clientId); + this.logger.close && this.logger.close(); + } + + async #doHeartbeat() { + const request = this.wrapHeartbeatRequest(); + for (const endpoints of this.getTotalRouteEndpoints()) { + await this.rpcClientManager.heartbeat(endpoints, request, this.requestTimeout); + } + } + + #getTotalRouteEndpointsMap() { + const endpointsMap = new Map(); + for (const topicRoute of this.topicRouteCache.values()) { + for (const endpoints of topicRoute.getTotalEndpoints()) { + endpointsMap.set(endpoints.facade, endpoints); + } + } + return endpointsMap; + } + + protected getTotalRouteEndpoints() { + const endpointsMap = this.#getTotalRouteEndpointsMap(); + return Array.from(endpointsMap.values()); + } + + protected findNewRouteEndpoints(endpointsList: Endpoints[]) { + const endpointsMap = this.#getTotalRouteEndpointsMap(); + const newEndpoints: Endpoints[] = []; + for (const endpoints of endpointsList) { + if (!endpointsMap.has(endpoints.facade)) { + newEndpoints.push(endpoints); + } + } + return newEndpoints; + } + + protected async updateRoutes() { + for (const topic of this.topics) { + await this.#fetchTopicRoute(topic); + } + } + + protected async getRouteData(topic: string) { + let topicRouteData = this.topicRouteCache.get(topic); + if (!topicRouteData) { + this.topics.add(topic); + topicRouteData = await this.#fetchTopicRoute(topic); + } + return topicRouteData; + } + + async #fetchTopicRoute(topic: string) { + const req = new QueryRouteRequest(); + req.setTopic(createResource(topic)); + req.setEndpoints(this.endpoints.toProtobuf()); + const response = await this.rpcClientManager.queryRoute(this.endpoints, req, this.requestTimeout); + StatusChecker.check(response.getStatus()?.toObject()); + const topicRouteData = new TopicRouteData(response.getMessageQueuesList()); + const newEndpoints = this.findNewRouteEndpoints(topicRouteData.getTotalEndpoints()); + for (const endpoints of newEndpoints) { + // sync current settings to new endpoints + this.getTelemetrySession(endpoints).syncSettings(); + } + this.topicRouteCache.set(topic, topicRouteData); + this.onTopicRouteDataUpdate(topic, topicRouteData); + debug('fetchTopicRoute topic=%o topicRouteData=%j', topic, topicRouteData); + return topicRouteData; + } + + #syncSettings() { + const command = this.settingsCommand(); + for (const endpoints of this.getTotalRouteEndpoints()) { + this.telemetry(endpoints, command); + } + } + + settingsCommand() { + const command = new TelemetryCommand(); + command.setSettings(this.getSettings().toProtobuf()); + return command; + } + + getTelemetrySession(endpoints: Endpoints) { + let session = this.#telemetrySessions.get(endpoints.facade); + if (!session) { + session = new TelemetrySession(this, endpoints, this.logger); + this.#telemetrySessions.set(endpoints.facade, session); + } + return session; + } + + createTelemetryStream(endpoints: Endpoints) { + const metadata = this.getRequestMetadata(); + return this.rpcClientManager.telemetry(endpoints, metadata); + } + + telemetry(endpoints: Endpoints, command: TelemetryCommand) { + this.getTelemetrySession(endpoints).write(command); + } + + getRequestMetadata() { + // https://github.com/apache/rocketmq-clients/blob/master/docs/transport.md + // Transport Header + const metadata = new Metadata(); + // version of protocol + metadata.set('x-mq-protocol', 'v2'); + // client unique identifier: mbp@78774@2@3549a8wsr + metadata.set('x-mq-client-id', this.clientId); + // current timestamp: 20210309T195445Z, DATE_TIME_FORMAT = "yyyyMMdd'T'HHmmss'Z'" + const dateTime = getRequestDateTime(); + metadata.set('x-mq-date-time', dateTime); + // request id for each gRPC header: f122a1e0-dbcf-4ca4-9db7-221903354be7 + metadata.set('x-mq-request-id', randomUUID()); + // language of client + // FIXME: java.lang.IllegalArgumentException: No enum constant org.apache.rocketmq.remoting.protocol.LanguageCode.nodejs + // https://github.com/apache/rocketmq/blob/master/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java + metadata.set('x-mq-language', 'HTTP'); + // version of client + metadata.set('x-mq-client-version', UserAgent.INSTANCE.version); + if (this.#sessionCredentials) { + if (this.#sessionCredentials.securityToken) { + metadata.set('x-mq-session-token', this.#sessionCredentials.securityToken); + } + const signature = sign(this.#sessionCredentials.accessSecret, dateTime); + const authorization = `MQv2-HMAC-SHA1 Credential=${this.#sessionCredentials.accessKey}, SignedHeaders=x-mq-date-time, Signature=${signature}`; + metadata.set('authorization', authorization); + } + return metadata; + } + + protected abstract getSettings(): Settings; + + /** + * Wrap heartbeat request + */ + protected abstract wrapHeartbeatRequest(): HeartbeatRequest; + + /** + * Wrap notify client termination request. + */ + protected abstract wrapNotifyClientTerminationRequest(): NotifyClientTerminationRequest; + + #releaseTelemetrySessions() { + for (const session of this.#telemetrySessions.values()) { + session.release(); + } + this.#telemetrySessions.clear(); + } + + /** + * Notify remote that current client is prepared to be terminated. + */ + async #notifyClientTermination() { + this.logger.info('Notify remote that client is terminated, clientId=%s', this.clientId); + const request = this.wrapNotifyClientTerminationRequest(); + for (const endpoints of this.getTotalRouteEndpoints()) { + await this.rpcClientManager.notifyClientTermination(endpoints, request, this.requestTimeout); + } + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + protected onTopicRouteDataUpdate(_topic: string, _topicRouteData: TopicRouteData) { + // sub class can monitor topic route data change here + } + + onUnknownCommand(endpoints: Endpoints, status: Status.AsObject) { + try { + StatusChecker.check(status); + } catch (err) { + this.logger.error('Get error status from telemetry session, status=%j, endpoints=%j, clientId=%s', + status, endpoints, this.clientId); + this.#startupReject && this.#startupReject(err as ClientException); + } + } + + onSettingsCommand(_endpoints: Endpoints, settings: SettingsPB) { + // final Metric metric = new Metric(settings.getMetric()); + // clientMeterManager.reset(metric); + this.getSettings().sync(settings); + this.logger.info('Sync settings=%j, clientId=%s', this.getSettings(), this.clientId); + this.#startupResolve && this.#startupResolve(); + } + + onRecoverOrphanedTransactionCommand(_endpoints: Endpoints, command: RecoverOrphanedTransactionCommand) { + this.logger.warn('Ignore orphaned transaction recovery command from remote, which is not expected, clientId=%s, command=%j', + this.clientId, command.toObject()); + // const telemetryCommand = new TelemetryCommand(); + // telemetryCommand.setStatus(new Status().setCode(Code.NOT_IMPLEMENTED)); + // telemetryCommand.setRecoverOrphanedTransactionCommand(new RecoverOrphanedTransactionCommand()); + // this.telemetry(endpoints, telemetryCommand); + } + + onVerifyMessageCommand(endpoints: Endpoints, command: VerifyMessageCommand) { + const obj = command.toObject(); + this.logger.warn('Ignore verify message command from remote, which is not expected, clientId=%s, command=%j', + this.clientId, obj); + const telemetryCommand = new TelemetryCommand(); + telemetryCommand.setStatus(new Status().setCode(Code.NOT_IMPLEMENTED)); + telemetryCommand.setVerifyMessageCommand(new VerifyMessageCommand().setNonce(obj.nonce)); + this.telemetry(endpoints, telemetryCommand); + } + + onPrintThreadStackTraceCommand(endpoints: Endpoints, command: PrintThreadStackTraceCommand) { + const obj = command.toObject(); + this.logger.warn('Ignore orphaned transaction recovery command from remote, which is not expected, clientId=%s, command=%j', + this.clientId, obj); + const nonce = obj.nonce; + const telemetryCommand = new TelemetryCommand(); + telemetryCommand.setThreadStackTrace(new ThreadStackTrace().setThreadStackTrace('mock stack').setNonce(nonce)); + telemetryCommand.setStatus(new Status().setCode(Code.OK)); + this.telemetry(endpoints, telemetryCommand); + } +} diff --git a/nodejs/src/client/ClientId.ts b/nodejs/src/client/ClientId.ts new file mode 100644 index 000000000..76cbfd19a --- /dev/null +++ b/nodejs/src/client/ClientId.ts @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { hostname } from 'node:os'; + +/** + * Client Identifier Helper + * https://github.com/apache/rocketmq-clients/blob/master/docs/design.md#client-identifier + */ +export class ClientId { + static #hostname = hostname(); + static #index = 0n; + + static create() { + return `${this.#hostname}@${process.pid}@${this.#index++}@${Date.now().toString(36)}`; + } +} diff --git a/nodejs/src/client/Logger.ts b/nodejs/src/client/Logger.ts new file mode 100644 index 000000000..8a5abbeb5 --- /dev/null +++ b/nodejs/src/client/Logger.ts @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'node:path'; +import { homedir } from 'node:os'; +import { EggLogger } from 'egg-logger'; + +export interface ILogger { + info(...args: any[]): void; + warn(...args: any[]): void; + error(...args: any[]): void; + close?(...args: any[]): void; +} + +export function getDefaultLogger() { + const file = path.join(homedir(), 'logs/rocketmq/rocketmq_client_nodejs.log'); + return new EggLogger({ + file, + level: 'INFO', + }); +} diff --git a/nodejs/src/client/RpcClient.ts b/nodejs/src/client/RpcClient.ts new file mode 100644 index 000000000..144bc5478 --- /dev/null +++ b/nodejs/src/client/RpcClient.ts @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ChannelCredentials, Metadata } from '@grpc/grpc-js'; +import { MessagingServiceClient } from '../../proto/apache/rocketmq/v2/service_grpc_pb'; +import { + AckMessageRequest, + AckMessageResponse, + ChangeInvisibleDurationRequest, + ChangeInvisibleDurationResponse, + EndTransactionRequest, + EndTransactionResponse, + ForwardMessageToDeadLetterQueueRequest, + ForwardMessageToDeadLetterQueueResponse, + GetOffsetRequest, + GetOffsetResponse, + HeartbeatRequest, + HeartbeatResponse, + NotifyClientTerminationRequest, + NotifyClientTerminationResponse, + PullMessageRequest, + PullMessageResponse, + QueryAssignmentRequest, + QueryAssignmentResponse, + QueryOffsetRequest, + QueryOffsetResponse, + QueryRouteRequest, + QueryRouteResponse, + ReceiveMessageRequest, + ReceiveMessageResponse, + SendMessageRequest, + SendMessageResponse, + UpdateOffsetRequest, + UpdateOffsetResponse, +} from '../../proto/apache/rocketmq/v2/service_pb'; +import { Endpoints } from '../route'; + +export class RpcClient { + #client: MessagingServiceClient; + #activityTime = Date.now(); + + constructor(endpoints: Endpoints, sslEnabled: boolean) { + const address = endpoints.getGrpcTarget(); + const grpcCredentials = sslEnabled ? ChannelCredentials.createSsl() : ChannelCredentials.createInsecure(); + this.#client = new MessagingServiceClient(address, grpcCredentials); + } + + #getAndActivityRpcClient() { + this.#activityTime = Date.now(); + return this.#client; + } + + #getDeadline(duration: number) { + return Date.now() + duration; + } + + idleDuration() { + return Date.now() - this.#activityTime; + } + + close() { + this.#client.close(); + } + + /** + * Query topic route + * + * @param request query route request. + * @param metadata gRPC request header metadata. + * @param duration request max duration in milliseconds. + */ + async queryRoute(request: QueryRouteRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.queryRoute(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async heartbeat(request: HeartbeatRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.heartbeat(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async sendMessage(request: SendMessageRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.sendMessage(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async queryAssignment(request: QueryAssignmentRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.queryAssignment(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async receiveMessage(request: ReceiveMessageRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + const readable = client.receiveMessage(request, metadata, { deadline }); + const responses: ReceiveMessageResponse[] = []; + for await (const res of readable) { + responses.push(res); + } + return responses; + } + + async ackMessage(request: AckMessageRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.ackMessage(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async forwardMessageToDeadLetterQueue(request: ForwardMessageToDeadLetterQueueRequest, + metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.forwardMessageToDeadLetterQueue(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async pullMessage(request: PullMessageRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + const readable = client.pullMessage(request, metadata, { deadline }); + const responses: PullMessageResponse[] = []; + for await (const res of readable) { + responses.push(res); + } + return responses; + } + + async updateOffset(request: UpdateOffsetRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.updateOffset(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async getOffset(request: GetOffsetRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.getOffset(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async queryOffset(request: QueryOffsetRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.queryOffset(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async endTransaction(request: EndTransactionRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.endTransaction(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + telemetry(metadata: Metadata) { + const client = this.#getAndActivityRpcClient(); + return client.telemetry(metadata); + } + + async notifyClientTermination(request: NotifyClientTerminationRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.notifyClientTermination(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } + + async changeInvisibleDuration(request: ChangeInvisibleDurationRequest, metadata: Metadata, duration: number) { + const client = this.#getAndActivityRpcClient(); + const deadline = this.#getDeadline(duration); + return new Promise((resolve, reject) => { + client.changeInvisibleDuration(request, metadata, { deadline }, (e, res) => { + if (e) return reject(e); + resolve(res); + }); + }); + } +} diff --git a/nodejs/src/client/RpcClientManager.ts b/nodejs/src/client/RpcClientManager.ts new file mode 100644 index 000000000..86405f606 --- /dev/null +++ b/nodejs/src/client/RpcClientManager.ts @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Metadata } from '@grpc/grpc-js'; +import { + AckMessageRequest, + ChangeInvisibleDurationRequest, + EndTransactionRequest, + ForwardMessageToDeadLetterQueueRequest, + GetOffsetRequest, + HeartbeatRequest, NotifyClientTerminationRequest, PullMessageRequest, + QueryAssignmentRequest, QueryOffsetRequest, QueryRouteRequest, + ReceiveMessageRequest, SendMessageRequest, UpdateOffsetRequest, +} from '../../proto/apache/rocketmq/v2/service_pb'; +import { Endpoints } from '../route'; +import { ILogger } from './Logger'; +import { RpcClient } from './RpcClient'; +import type { BaseClient } from './BaseClient'; + +const RPC_CLIENT_MAX_IDLE_DURATION = 30 * 60000; // 30 minutes +const RPC_CLIENT_IDLE_CHECK_PERIOD = 60000; + +export class RpcClientManager { + #rpcClients = new Map(); + #baseClient: BaseClient; + #logger: ILogger; + #clearIdleRpcClientsTimer: NodeJS.Timeout; + + constructor(baseClient: BaseClient, logger: ILogger) { + this.#baseClient = baseClient; + this.#logger = logger; + this.#startUp(); + } + + #startUp() { + this.#clearIdleRpcClientsTimer = setInterval(() => { + this.#clearIdleRpcClients(); + }, RPC_CLIENT_IDLE_CHECK_PERIOD); + } + + #clearIdleRpcClients() { + for (const [ endpoints, rpcClient ] of this.#rpcClients.entries()) { + const idleDuration = rpcClient.idleDuration(); + if (idleDuration > RPC_CLIENT_MAX_IDLE_DURATION) { + rpcClient.close(); + this.#rpcClients.delete(endpoints); + this.#logger.info('[RpcClientManager] Rpc client has been idle for a long time, endpoints=%s, idleDuration=%s, clientId=%s', + endpoints, idleDuration, RPC_CLIENT_MAX_IDLE_DURATION, this.#baseClient.clientId); + } + } + } + + #getRpcClient(endpoints: Endpoints) { + let rpcClient = this.#rpcClients.get(endpoints); + if (!rpcClient) { + rpcClient = new RpcClient(endpoints, this.#baseClient.sslEnabled); + this.#rpcClients.set(endpoints, rpcClient); + } + return rpcClient; + } + + close() { + for (const [ endpoints, rpcClient ] of this.#rpcClients.entries()) { + rpcClient.close(); + this.#rpcClients.delete(endpoints); + } + clearInterval(this.#clearIdleRpcClientsTimer); + } + + async queryRoute(endpoints: Endpoints, request: QueryRouteRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.queryRoute(request, metadata, duration); + } + + async heartbeat(endpoints: Endpoints, request: HeartbeatRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.heartbeat(request, metadata, duration); + } + + async sendMessage(endpoints: Endpoints, request: SendMessageRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.sendMessage(request, metadata, duration); + } + + async queryAssignment(endpoints: Endpoints, request: QueryAssignmentRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.queryAssignment(request, metadata, duration); + } + + async receiveMessage(endpoints: Endpoints, request: ReceiveMessageRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.receiveMessage(request, metadata, duration); + } + + async ackMessage(endpoints: Endpoints, request: AckMessageRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.ackMessage(request, metadata, duration); + } + + async forwardMessageToDeadLetterQueue(endpoints: Endpoints, request: ForwardMessageToDeadLetterQueueRequest, + duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.forwardMessageToDeadLetterQueue(request, metadata, duration); + } + + async pullMessage(endpoints: Endpoints, request: PullMessageRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.pullMessage(request, metadata, duration); + } + + async updateOffset(endpoints: Endpoints, request: UpdateOffsetRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.updateOffset(request, metadata, duration); + } + + async getOffset(endpoints: Endpoints, request: GetOffsetRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.getOffset(request, metadata, duration); + } + + async queryOffset(endpoints: Endpoints, request: QueryOffsetRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.queryOffset(request, metadata, duration); + } + + async endTransaction(endpoints: Endpoints, request: EndTransactionRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.endTransaction(request, metadata, duration); + } + + telemetry(endpoints: Endpoints, metadata: Metadata) { + const rpcClient = this.#getRpcClient(endpoints); + return rpcClient.telemetry(metadata); + } + + async changeInvisibleDuration(endpoints: Endpoints, request: ChangeInvisibleDurationRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.changeInvisibleDuration(request, metadata, duration); + } + + async notifyClientTermination(endpoints: Endpoints, request: NotifyClientTerminationRequest, duration: number) { + const rpcClient = this.#getRpcClient(endpoints); + const metadata = this.#baseClient.getRequestMetadata(); + return await rpcClient.notifyClientTermination(request, metadata, duration); + } +} diff --git a/nodejs/src/client/SessionCredentials.ts b/nodejs/src/client/SessionCredentials.ts new file mode 100644 index 000000000..e0d2e17b5 --- /dev/null +++ b/nodejs/src/client/SessionCredentials.ts @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface SessionCredentials { + accessKey: string; + accessSecret: string; + securityToken?: string; +} diff --git a/nodejs/src/client/Settings.ts b/nodejs/src/client/Settings.ts new file mode 100644 index 000000000..34f7eebdc --- /dev/null +++ b/nodejs/src/client/Settings.ts @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientType, Settings as SettingsPB } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { Endpoints } from '../route/Endpoints'; +import { RetryPolicy } from '../retry'; + +export abstract class Settings { + protected readonly clientId: string; + protected readonly clientType: ClientType; + protected readonly accessPoint: Endpoints; + protected retryPolicy?: RetryPolicy; + protected readonly requestTimeout: number; + + constructor(clientId: string, clientType: ClientType, accessPoint: Endpoints, requestTimeout: number, + retryPolicy?: RetryPolicy) { + this.clientId = clientId; + this.clientType = clientType; + this.accessPoint = accessPoint; + this.retryPolicy = retryPolicy; + this.requestTimeout = requestTimeout; + } + + abstract toProtobuf(): SettingsPB; + + abstract sync(settings: SettingsPB): void; + + getRetryPolicy() { + return this.retryPolicy; + } +} diff --git a/nodejs/src/client/TelemetrySession.ts b/nodejs/src/client/TelemetrySession.ts new file mode 100644 index 000000000..3812f928f --- /dev/null +++ b/nodejs/src/client/TelemetrySession.ts @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientDuplexStream } from '@grpc/grpc-js'; +import { TelemetryCommand } from '../../proto/apache/rocketmq/v2/service_pb'; +import { Endpoints } from '../route'; +import type { BaseClient } from './BaseClient'; +import { ILogger } from './Logger'; + +export class TelemetrySession { + #endpoints: Endpoints; + #baseClient: BaseClient; + #logger: ILogger; + #stream: ClientDuplexStream; + + constructor(baseClient: BaseClient, endpoints: Endpoints, logger: ILogger) { + this.#endpoints = endpoints; + this.#baseClient = baseClient; + this.#logger = logger; + this.#renewStream(true); + } + + release() { + this.#logger.info('Begin to release telemetry session, endpoints=%s, clientId=%s', + this.#endpoints, this.#baseClient.clientId); + this.#stream.end(); + this.#stream.removeAllListeners(); + } + + write(command: TelemetryCommand) { + this.#stream.write(command); + } + + syncSettings() { + const command = this.#baseClient.settingsCommand(); + this.write(command); + } + + #renewStream(inited: boolean) { + this.#stream = this.#baseClient.createTelemetryStream(this.#endpoints); + this.#stream.on('data', this.#onData.bind(this)); + this.#stream.once('error', this.#onError.bind(this)); + this.#stream.once('end', this.#onEnd.bind(this)); + if (!inited) { + this.syncSettings(); + } + } + + #onData(command: TelemetryCommand) { + const endpoints = this.#endpoints; + const clientId = this.#baseClient.clientId; + const commandCase = command.getCommandCase(); + switch (commandCase) { + case TelemetryCommand.CommandCase.SETTINGS: + this.#logger.info('Receive settings from remote, endpoints=%s, clientId=%s', + endpoints, clientId); + this.#baseClient.onSettingsCommand(endpoints, command.getSettings()!); + break; + case TelemetryCommand.CommandCase.RECOVER_ORPHANED_TRANSACTION_COMMAND: { + this.#logger.info('Receive orphaned transaction recovery command from remote, endpoints=%s, clientId=%s', + endpoints, clientId); + this.#baseClient.onRecoverOrphanedTransactionCommand(endpoints, command.getRecoverOrphanedTransactionCommand()!); + break; + } + case TelemetryCommand.CommandCase.VERIFY_MESSAGE_COMMAND: { + this.#logger.info('Receive message verification command from remote, endpoints=%s, clientId=%s', + endpoints, clientId); + this.#baseClient.onVerifyMessageCommand(endpoints, command.getVerifyMessageCommand()!); + break; + } + case TelemetryCommand.CommandCase.PRINT_THREAD_STACK_TRACE_COMMAND: { + this.#logger.info('Receive thread stack print command from remote, endpoints=%s, clientId=%s', + endpoints, clientId); + this.#baseClient.onPrintThreadStackTraceCommand(endpoints, command.getPrintThreadStackTraceCommand()!); + break; + } + default: { + const commandObj = command.toObject(); + this.#logger.warn('Receive unrecognized command from remote, endpoints=%s, commandCase=%j, command=%j, clientId=%s', + endpoints, commandCase, commandObj, clientId); + // should telemetry session start fail + this.#baseClient.onUnknownCommand(endpoints, commandObj.status!); + } + } + } + + #onError(err: Error) { + this.#logger.error('Exception raised from stream response observer, endpoints=%s, clientId=%s, error=%s', + this.#endpoints, this.#baseClient.clientId, err); + this.release(); + setTimeout(() => { + this.#renewStream(false); + }, 1000); + } + + #onEnd() { + this.#logger.info('Receive completion for stream response observer, endpoints=%s, clientId=%s', + this.#endpoints, this.#baseClient.clientId); + this.release(); + setTimeout(() => { + this.#renewStream(false); + }, 1000); + } +} diff --git a/nodejs/src/client/UserAgent.ts b/nodejs/src/client/UserAgent.ts new file mode 100644 index 000000000..d0310ad9b --- /dev/null +++ b/nodejs/src/client/UserAgent.ts @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'node:os'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; +import { UA, Language } from '../../proto/apache/rocketmq/v2/definition_pb'; + +const VERSION: string = JSON.parse(readFileSync(path.join(__dirname, '../../package.json'), 'utf-8')).version; + +export class UserAgent { + static readonly INSTANCE = new UserAgent(VERSION, os.platform(), os.hostname()); + + readonly version: string; + readonly platform: string; + readonly hostname: string; + + constructor(version: string, platform: string, hostname: string) { + this.version = version; + this.platform = platform; + this.hostname = hostname; + } + + toProtobuf() { + return new UA() + .setLanguage(Language.NODE_JS) + .setVersion(this.version) + .setPlatform(this.platform) + .setHostname(this.hostname); + } +} diff --git a/nodejs/src/client/index.ts b/nodejs/src/client/index.ts new file mode 100644 index 000000000..c600050d6 --- /dev/null +++ b/nodejs/src/client/index.ts @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './BaseClient'; +export * from './ClientId'; +export * from './Logger'; +export * from './UserAgent'; +export * from './RpcClient'; +export * from './RpcClientManager'; +export * from './SessionCredentials'; +export * from './Settings'; +export * from './TelemetrySession'; +export * from './UserAgent'; diff --git a/nodejs/src/consumer/Consumer.ts b/nodejs/src/consumer/Consumer.ts new file mode 100644 index 000000000..cdc78bf3c --- /dev/null +++ b/nodejs/src/consumer/Consumer.ts @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Message, Status } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { + AckMessageRequest, + ChangeInvisibleDurationRequest, + ReceiveMessageRequest, ReceiveMessageResponse, +} from '../../proto/apache/rocketmq/v2/service_pb'; +import { MessageView } from '../message'; +import { MessageQueue } from '../route'; +import { StatusChecker } from '../exception'; +import { BaseClient, BaseClientOptions } from '../client'; +import { createDuration, createResource } from '../util'; +import { FilterExpression } from './FilterExpression'; + +export interface ConsumerOptions extends BaseClientOptions { + consumerGroup: string; +} + +export abstract class Consumer extends BaseClient { + protected readonly consumerGroup: string; + + constructor(options: ConsumerOptions) { + super(options); + this.consumerGroup = options.consumerGroup; + } + + protected wrapReceiveMessageRequest(batchSize: number, mq: MessageQueue, + filterExpression: FilterExpression, invisibleDuration: number, longPollingTimeout: number) { + return new ReceiveMessageRequest() + .setGroup(createResource(this.consumerGroup)) + .setMessageQueue(mq.toProtobuf()) + .setFilterExpression(filterExpression.toProtobuf()) + .setLongPollingTimeout(createDuration(longPollingTimeout)) + .setBatchSize(batchSize) + .setAutoRenew(false) + .setInvisibleDuration(createDuration(invisibleDuration)); + } + + protected async receiveMessage(request: ReceiveMessageRequest, mq: MessageQueue, awaitDuration: number) { + const endpoints = mq.broker.endpoints; + const timeout = this.requestTimeout + awaitDuration; + let status: Status.AsObject | undefined; + const responses = await this.rpcClientManager.receiveMessage(endpoints, request, timeout); + const messageList: Message[] = []; + let transportDeliveryTimestamp: Date | undefined; + for (const response of responses) { + switch (response.getContentCase()) { + case ReceiveMessageResponse.ContentCase.STATUS: + status = response.getStatus()?.toObject(); + break; + case ReceiveMessageResponse.ContentCase.MESSAGE: + messageList.push(response.getMessage()!); + break; + case ReceiveMessageResponse.ContentCase.DELIVERY_TIMESTAMP: + transportDeliveryTimestamp = response.getDeliveryTimestamp()?.toDate(); + break; + default: + // this.logger.warn("[Bug] Not recognized content for receive message response, mq={}, " + + // "clientId={}, response={}", mq, clientId, response); + } + } + StatusChecker.check(status); + const messages = messageList.map(message => new MessageView(message, mq, transportDeliveryTimestamp)); + return messages; + } + + protected async ackMessage(messageView: MessageView) { + const endpoints = messageView.endpoints; + const request = new AckMessageRequest() + .setGroup(createResource(this.consumerGroup)) + .setTopic(createResource(messageView.topic)); + request.addEntries() + .setMessageId(messageView.messageId) + .setReceiptHandle(messageView.receiptHandle); + const res = await this.rpcClientManager.ackMessage(endpoints, request, this.requestTimeout); + // FIXME: handle fail ack + const response = res.toObject(); + StatusChecker.check(response.status); + return response.entriesList; + } + + protected async invisibleDuration(messageView: MessageView, invisibleDuration: number) { + const request = new ChangeInvisibleDurationRequest() + .setGroup(createResource(this.consumerGroup)) + .setTopic(createResource(messageView.topic)) + .setReceiptHandle(messageView.receiptHandle) + .setInvisibleDuration(createDuration(invisibleDuration)) + .setMessageId(messageView.messageId); + + const res = await this.rpcClientManager.changeInvisibleDuration(messageView.endpoints, request, this.requestTimeout); + const response = res.toObject(); + StatusChecker.check(response.status); + return response.receiptHandle; + } +} diff --git a/nodejs/src/consumer/FilterExpression.ts b/nodejs/src/consumer/FilterExpression.ts new file mode 100644 index 000000000..e050e8cc4 --- /dev/null +++ b/nodejs/src/consumer/FilterExpression.ts @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FilterType, FilterExpression as FilterExpressionPB } from '../../proto/apache/rocketmq/v2/definition_pb'; + +const TAG_EXPRESSION_SUB_ALL = '*'; + +export class FilterExpression { + static readonly SUB_ALL = new FilterExpression(TAG_EXPRESSION_SUB_ALL); + + readonly expression: string; + readonly filterType: FilterType; + + constructor(expression: string, filterType = FilterType.TAG) { + this.expression = expression; + this.filterType = filterType; + } + + toProtobuf() { + return new FilterExpressionPB() + .setType(this.filterType) + .setExpression(this.expression); + } + + toString() { + return `FilterExpression(${this.filterType},${this.expression})`; + } +} diff --git a/nodejs/src/consumer/SimpleConsumer.ts b/nodejs/src/consumer/SimpleConsumer.ts new file mode 100644 index 000000000..916f362cb --- /dev/null +++ b/nodejs/src/consumer/SimpleConsumer.ts @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientType } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { HeartbeatRequest, NotifyClientTerminationRequest } from '../../proto/apache/rocketmq/v2/service_pb'; +import { MessageView } from '../message'; +import { TopicRouteData } from '../route'; +import { createResource } from '../util'; +import { FilterExpression } from './FilterExpression'; +import { SimpleSubscriptionSettings } from './SimpleSubscriptionSettings'; +import { SubscriptionLoadBalancer } from './SubscriptionLoadBalancer'; +import { Consumer, ConsumerOptions } from './Consumer'; + +export interface SimpleConsumerOptions extends ConsumerOptions { + /** + * support tag string as filter, e.g.: + * ```ts + * new Map() + * .set('TestTopic1', 'TestTag1') + * .set('TestTopic2', 'TestTag2') + * ``` + */ + subscriptions: Map; + /** + * set await duration for long-polling, default is 30000ms + */ + awaitDuration?: number; +} + +export class SimpleConsumer extends Consumer { + readonly #simpleSubscriptionSettings: SimpleSubscriptionSettings; + readonly #subscriptionExpressions = new Map(); + readonly #subscriptionRouteDataCache = new Map(); + readonly #awaitDuration: number; + #topicIndex = 0; + + constructor(options: SimpleConsumerOptions) { + options.topics = Array.from(options.subscriptions.keys()); + super(options); + for (const [ topic, filter ] of options.subscriptions.entries()) { + if (typeof filter === 'string') { + // filter is tag string + this.#subscriptionExpressions.set(topic, new FilterExpression(filter)); + } else { + this.#subscriptionExpressions.set(topic, filter); + } + } + this.#awaitDuration = options.awaitDuration ?? 30000; + this.#simpleSubscriptionSettings = new SimpleSubscriptionSettings(this.clientId, this.endpoints, + this.consumerGroup, this.requestTimeout, this.#awaitDuration, this.#subscriptionExpressions); + } + + protected getSettings() { + return this.#simpleSubscriptionSettings; + } + + protected wrapHeartbeatRequest() { + return new HeartbeatRequest() + .setClientType(ClientType.SIMPLE_CONSUMER) + .setGroup(createResource(this.consumerGroup)); + } + + protected wrapNotifyClientTerminationRequest() { + return new NotifyClientTerminationRequest() + .setGroup(createResource(this.consumerGroup)); + } + + protected onTopicRouteDataUpdate(topic: string, topicRouteData: TopicRouteData) { + this.#updateSubscriptionLoadBalancer(topic, topicRouteData); + } + + #updateSubscriptionLoadBalancer(topic: string, topicRouteData: TopicRouteData) { + let subscriptionLoadBalancer = this.#subscriptionRouteDataCache.get(topic); + if (!subscriptionLoadBalancer) { + subscriptionLoadBalancer = new SubscriptionLoadBalancer(topicRouteData); + } else { + subscriptionLoadBalancer = subscriptionLoadBalancer.update(topicRouteData); + } + this.#subscriptionRouteDataCache.set(topic, subscriptionLoadBalancer); + return subscriptionLoadBalancer; + } + + async #getSubscriptionLoadBalancer(topic: string) { + let loadBalancer = this.#subscriptionRouteDataCache.get(topic); + if (!loadBalancer) { + const topicRouteData = await this.getRouteData(topic); + loadBalancer = this.#updateSubscriptionLoadBalancer(topic, topicRouteData); + } + return loadBalancer; + } + + async subscribe(topic: string, filterExpression: FilterExpression) { + await this.getRouteData(topic); + this.#subscriptionExpressions.set(topic, filterExpression); + } + + unsubscribe(topic: string) { + this.#subscriptionExpressions.delete(topic); + } + + async receive(maxMessageNum = 10, invisibleDuration = 15000) { + const topic = this.#nextTopic(); + const filterExpression = this.#subscriptionExpressions.get(topic)!; + const loadBalancer = await this.#getSubscriptionLoadBalancer(topic); + const mq = loadBalancer.takeMessageQueue(); + const request = this.wrapReceiveMessageRequest(maxMessageNum, mq, filterExpression, + invisibleDuration, this.#awaitDuration); + return await this.receiveMessage(request, mq, this.#awaitDuration); + } + + async ack(message: MessageView) { + await this.ackMessage(message); + } + + async changeInvisibleDuration0(message: MessageView, invisibleDuration: number) { + await this.changeInvisibleDuration0(message, invisibleDuration); + } + + #nextTopic() { + const topics = Array.from(this.#subscriptionExpressions.keys()); + if (this.#topicIndex >= topics.length) { + this.#topicIndex = 0; + } + return topics[this.#topicIndex++]; + } +} diff --git a/nodejs/src/consumer/SimpleSubscriptionSettings.ts b/nodejs/src/consumer/SimpleSubscriptionSettings.ts new file mode 100644 index 000000000..116a613bc --- /dev/null +++ b/nodejs/src/consumer/SimpleSubscriptionSettings.ts @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Settings as SettingsPB, + ClientType, + Subscription, +} from '../../proto/apache/rocketmq/v2/definition_pb'; +import { Endpoints } from '../route'; +import { Settings, UserAgent } from '../client'; +import { createDuration, createResource } from '../util'; +import { FilterExpression } from './FilterExpression'; + +export class SimpleSubscriptionSettings extends Settings { + readonly longPollingTimeout: number; + readonly group: string; + readonly subscriptionExpressions: Map; + + constructor(clientId: string, accessPoint: Endpoints, group: string, + requestTimeout: number, longPollingTimeout: number, subscriptionExpressions: Map) { + super(clientId, ClientType.SIMPLE_CONSUMER, accessPoint, requestTimeout); + this.longPollingTimeout = longPollingTimeout; + this.group = group; + this.subscriptionExpressions = subscriptionExpressions; + } + + toProtobuf(): SettingsPB { + const subscription = new Subscription() + .setGroup(createResource(this.group)) + .setLongPollingTimeout(createDuration(this.longPollingTimeout)); + + for (const [ topic, filterExpression ] of this.subscriptionExpressions.entries()) { + subscription.addSubscriptions() + .setTopic(createResource(topic)) + .setExpression(filterExpression.toProtobuf()); + } + return new SettingsPB() + .setClientType(this.clientType) + .setAccessPoint(this.accessPoint.toProtobuf()) + .setRequestTimeout(createDuration(this.requestTimeout)) + .setSubscription(subscription) + .setUserAgent(UserAgent.INSTANCE.toProtobuf()); + } + + sync(settings: SettingsPB): void { + if (settings.getPubSubCase() !== SettingsPB.PubSubCase.SUBSCRIPTION) { + // log.error("[Bug] Issued settings not match with the client type, clientId={}, pubSubCase={}, " + // + "clientType={}", clientId, pubSubCase, clientType); + } + } +} diff --git a/nodejs/src/consumer/SubscriptionLoadBalancer.ts b/nodejs/src/consumer/SubscriptionLoadBalancer.ts new file mode 100644 index 000000000..f191f24a3 --- /dev/null +++ b/nodejs/src/consumer/SubscriptionLoadBalancer.ts @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { randomInt } from 'node:crypto'; +import { Permission } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { MessageQueue, TopicRouteData } from '../route'; +import { MASTER_BROKER_ID } from '../util'; + +export class SubscriptionLoadBalancer { + #index: number; + #messageQueues: MessageQueue[]; + + constructor(topicRouteData: TopicRouteData, index?: number) { + this.#messageQueues = topicRouteData.messageQueues.filter(mq => { + return mq.queueId === MASTER_BROKER_ID && (mq.permission === Permission.READ || mq.permission === Permission.READ_WRITE); + }); + this.#index = index === undefined ? randomInt(this.#messageQueues.length) : index; + if (this.#messageQueues.length === 0) { + throw new Error(`No readable message queue found, topicRouteData=${JSON.stringify(topicRouteData)}`); + } + } + + update(topicRouteData: TopicRouteData) { + return new SubscriptionLoadBalancer(topicRouteData, this.#index); + } + + takeMessageQueue() { + if (this.#index >= this.#messageQueues.length) { + this.#index = 0; + } + return this.#messageQueues[this.#index++]; + } +} diff --git a/nodejs/src/consumer/index.ts b/nodejs/src/consumer/index.ts new file mode 100644 index 000000000..73dbd62fa --- /dev/null +++ b/nodejs/src/consumer/index.ts @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './Consumer'; +export * from './FilterExpression'; +export * from './SimpleConsumer'; +export * from './SimpleSubscriptionSettings'; +export * from './SubscriptionLoadBalancer'; diff --git a/nodejs/src/exception/BadRequestException.ts b/nodejs/src/exception/BadRequestException.ts new file mode 100644 index 000000000..bdb80c50c --- /dev/null +++ b/nodejs/src/exception/BadRequestException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class BadRequestException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'BadRequestException'; + } +} diff --git a/nodejs/src/exception/ClientException.ts b/nodejs/src/exception/ClientException.ts new file mode 100644 index 000000000..9c9ad5583 --- /dev/null +++ b/nodejs/src/exception/ClientException.ts @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const REQUEST_ID_KEY = 'request-id'; +const RESPONSE_CODE_KEY = 'response-code'; + +export class ClientException extends Error { + code: number; + + constructor(code: number, message: string, requestId?: string) { + super(`[${REQUEST_ID_KEY}=${requestId}, ${RESPONSE_CODE_KEY}=${code}] ${message}`); + this.code = code; + this.name = 'ClientException'; + } +} diff --git a/nodejs/src/exception/ForbiddenException.ts b/nodejs/src/exception/ForbiddenException.ts new file mode 100644 index 000000000..95eb95116 --- /dev/null +++ b/nodejs/src/exception/ForbiddenException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class ForbiddenException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'ForbiddenException'; + } +} diff --git a/nodejs/src/exception/InternalErrorException.ts b/nodejs/src/exception/InternalErrorException.ts new file mode 100644 index 000000000..accb043a8 --- /dev/null +++ b/nodejs/src/exception/InternalErrorException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class InternalErrorException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'InternalErrorException'; + } +} diff --git a/nodejs/src/exception/NotFoundException.ts b/nodejs/src/exception/NotFoundException.ts new file mode 100644 index 000000000..dcec7a87f --- /dev/null +++ b/nodejs/src/exception/NotFoundException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class NotFoundException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'NotFoundException'; + } +} diff --git a/nodejs/src/exception/PayloadTooLargeException.ts b/nodejs/src/exception/PayloadTooLargeException.ts new file mode 100644 index 000000000..7884a1bb9 --- /dev/null +++ b/nodejs/src/exception/PayloadTooLargeException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class PayloadTooLargeException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'PayloadTooLargeException'; + } +} diff --git a/nodejs/src/exception/PaymentRequiredException.ts b/nodejs/src/exception/PaymentRequiredException.ts new file mode 100644 index 000000000..664d06e18 --- /dev/null +++ b/nodejs/src/exception/PaymentRequiredException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class PaymentRequiredException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'PaymentRequiredException'; + } +} diff --git a/nodejs/src/exception/ProxyTimeoutException.ts b/nodejs/src/exception/ProxyTimeoutException.ts new file mode 100644 index 000000000..61686c4ea --- /dev/null +++ b/nodejs/src/exception/ProxyTimeoutException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class ProxyTimeoutException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'ProxyTimeoutException'; + } +} diff --git a/nodejs/src/exception/RequestHeaderFieldsTooLargeException.ts b/nodejs/src/exception/RequestHeaderFieldsTooLargeException.ts new file mode 100644 index 000000000..06e1a2e77 --- /dev/null +++ b/nodejs/src/exception/RequestHeaderFieldsTooLargeException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class RequestHeaderFieldsTooLargeException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'RequestHeaderFieldsTooLargeException'; + } +} diff --git a/nodejs/src/exception/StatusChecker.ts b/nodejs/src/exception/StatusChecker.ts new file mode 100644 index 000000000..d2f08a931 --- /dev/null +++ b/nodejs/src/exception/StatusChecker.ts @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Status, Code } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { BadRequestException } from './BadRequestException'; +import { ForbiddenException } from './ForbiddenException'; +import { InternalErrorException } from './InternalErrorException'; +import { NotFoundException } from './NotFoundException'; +import { PayloadTooLargeException } from './PayloadTooLargeException'; +import { PaymentRequiredException } from './PaymentRequiredException'; +import { ProxyTimeoutException } from './ProxyTimeoutException'; +import { RequestHeaderFieldsTooLargeException } from './RequestHeaderFieldsTooLargeException'; +import { TooManyRequestsException } from './TooManyRequestsException'; +import { UnauthorizedException } from './UnauthorizedException'; +import { UnsupportedException } from './UnsupportedException'; + +export class StatusChecker { + static check(status?: Status.AsObject, requestId?: string) { + if (!status) return; + switch (status.code) { + case Code.OK: + case Code.MULTIPLE_RESULTS: + return; + case Code.BAD_REQUEST: + case Code.ILLEGAL_ACCESS_POINT: + case Code.ILLEGAL_TOPIC: + case Code.ILLEGAL_CONSUMER_GROUP: + case Code.ILLEGAL_MESSAGE_TAG: + case Code.ILLEGAL_MESSAGE_KEY: + case Code.ILLEGAL_MESSAGE_GROUP: + case Code.ILLEGAL_MESSAGE_PROPERTY_KEY: + case Code.INVALID_TRANSACTION_ID: + case Code.ILLEGAL_MESSAGE_ID: + case Code.ILLEGAL_FILTER_EXPRESSION: + case Code.ILLEGAL_INVISIBLE_TIME: + case Code.ILLEGAL_DELIVERY_TIME: + case Code.INVALID_RECEIPT_HANDLE: + case Code.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE: + case Code.UNRECOGNIZED_CLIENT_TYPE: + case Code.MESSAGE_CORRUPTED: + case Code.CLIENT_ID_REQUIRED: + case Code.ILLEGAL_POLLING_TIME: + throw new BadRequestException(status.code, status.message, requestId); + case Code.UNAUTHORIZED: + throw new UnauthorizedException(status.code, status.message, requestId); + case Code.PAYMENT_REQUIRED: + throw new PaymentRequiredException(status.code, status.message, requestId); + case Code.FORBIDDEN: + throw new ForbiddenException(status.code, status.message, requestId); + case Code.MESSAGE_NOT_FOUND: + return; + case Code.NOT_FOUND: + case Code.TOPIC_NOT_FOUND: + case Code.CONSUMER_GROUP_NOT_FOUND: + throw new NotFoundException(status.code, status.message, requestId); + case Code.PAYLOAD_TOO_LARGE: + case Code.MESSAGE_BODY_TOO_LARGE: + throw new PayloadTooLargeException(status.code, status.message, requestId); + case Code.TOO_MANY_REQUESTS: + throw new TooManyRequestsException(status.code, status.message, requestId); + case Code.REQUEST_HEADER_FIELDS_TOO_LARGE: + case Code.MESSAGE_PROPERTIES_TOO_LARGE: + throw new RequestHeaderFieldsTooLargeException(status.code, status.message, requestId); + case Code.INTERNAL_ERROR: + case Code.INTERNAL_SERVER_ERROR: + case Code.HA_NOT_AVAILABLE: + throw new InternalErrorException(status.code, status.message, requestId); + case Code.PROXY_TIMEOUT: + case Code.MASTER_PERSISTENCE_TIMEOUT: + case Code.SLAVE_PERSISTENCE_TIMEOUT: + throw new ProxyTimeoutException(status.code, status.message, requestId); + case Code.UNSUPPORTED: + case Code.VERSION_UNSUPPORTED: + case Code.VERIFY_FIFO_MESSAGE_UNSUPPORTED: + throw new UnsupportedException(status.code, status.message, requestId); + default: + throw new UnsupportedException(status.code, status.message, requestId); + } + } +} diff --git a/nodejs/src/exception/TooManyRequestsException.ts b/nodejs/src/exception/TooManyRequestsException.ts new file mode 100644 index 000000000..ffcd9fe28 --- /dev/null +++ b/nodejs/src/exception/TooManyRequestsException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class TooManyRequestsException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'TooManyRequestsException'; + } +} diff --git a/nodejs/src/exception/UnauthorizedException.ts b/nodejs/src/exception/UnauthorizedException.ts new file mode 100644 index 000000000..b9de1b51b --- /dev/null +++ b/nodejs/src/exception/UnauthorizedException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class UnauthorizedException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'UnauthorizedException'; + } +} diff --git a/nodejs/src/exception/UnsupportedException.ts b/nodejs/src/exception/UnsupportedException.ts new file mode 100644 index 000000000..77b661cc5 --- /dev/null +++ b/nodejs/src/exception/UnsupportedException.ts @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientException } from './ClientException'; + +export class UnsupportedException extends ClientException { + constructor(code: number, message: string, requestId?: string) { + super(code, message, requestId); + this.name = 'UnsupportedException'; + } +} diff --git a/nodejs/src/exception/index.ts b/nodejs/src/exception/index.ts new file mode 100644 index 000000000..a40b59f40 --- /dev/null +++ b/nodejs/src/exception/index.ts @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './BadRequestException'; +export * from './ClientException'; +export * from './ForbiddenException'; +export * from './InternalErrorException'; +export * from './NotFoundException'; +export * from './PayloadTooLargeException'; +export * from './PaymentRequiredException'; +export * from './ProxyTimeoutException'; +export * from './RequestHeaderFieldsTooLargeException'; +export * from './StatusChecker'; +export * from './TooManyRequestsException'; +export * from './UnauthorizedException'; +export * from './UnsupportedException'; diff --git a/nodejs/src/index.ts b/nodejs/src/index.ts new file mode 100644 index 000000000..c36d39fac --- /dev/null +++ b/nodejs/src/index.ts @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './consumer'; +export * from './exception'; +export * from './message'; +export * from './producer'; +export * from './retry'; +export * from './route'; diff --git a/nodejs/src/message/Message.ts b/nodejs/src/message/Message.ts new file mode 100644 index 000000000..d380d90db --- /dev/null +++ b/nodejs/src/message/Message.ts @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface MessageOptions { + topic: string; + body: Buffer; + tag?: string; + messageGroup?: string; + keys?: string[]; + properties?: Map; + delay?: number; + deliveryTimestamp?: Date; +} + +export class Message { + topic: string; + body: Buffer; + tag?: string; + messageGroup?: string; + keys: string[]; + properties?: Map; + deliveryTimestamp?: Date; + + constructor(options: MessageOptions) { + this.topic = options.topic; + this.body = options.body; + this.tag = options.tag; + this.messageGroup = options.messageGroup; + this.keys = options.keys ?? []; + this.properties = options.properties; + let deliveryTimestamp = options.deliveryTimestamp; + if (options.delay && !deliveryTimestamp) { + deliveryTimestamp = new Date(Date.now() + options.delay); + } + this.deliveryTimestamp = deliveryTimestamp; + } +} diff --git a/nodejs/src/message/MessageId.ts b/nodejs/src/message/MessageId.ts new file mode 100644 index 000000000..c6523309d --- /dev/null +++ b/nodejs/src/message/MessageId.ts @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import address from 'address'; + +export enum MESSAGE_VERSION { + V0 = 0x00, + V1 = 0x01, +} + +export class MessageId { + id: string; + /** + * e.g.: 0x01: fixed 1 byte for current version + * offset = 0 + */ + version: MESSAGE_VERSION; + /** + * e.g.: 0x56F7E71C361B: lower 6 bytes of local mac address + * offset = 1 + */ + macAddress: string; + /** + * e.g.: 0x21BC: lower 2 bytes of process id + * offset = 7 + */ + processId: number; + /** + * e.g: 0x024CCDBE: seconds since 2021-01-01 00:00:00(UTC+0, lower 4 bytes) + * offset = 9 + */ + timestamp: number; + /** + * e.g.: 0x00000000: sequence number(4 bytes) + * offset = 13 + */ + sequence: number; + + toString() { + return this.id; + } +} + +const MAX_UINT32 = 0xFFFFFFFF; +const MAX_UINT16 = 0xFFFF; + +/** + * Message Identifier + * https://github.com/apache/rocketmq-clients/blob/master/docs/message_id.md + */ +export class MessageIdFactory { + // static #hostname = hostname(); + static #sequence = 0; + static #buf = Buffer.alloc(1 + 6 + 2 + 4 + 4); + // 2021-01-01 00:00:00(UTC+0), 1609459200000 + static #sinceTimestamp = new Date('2021-01-01T00:00:00Z').getTime() / 1000; + // lower 2 bytes of process id + static #processId = process.pid % MAX_UINT16; + static MAC = '000000000000'; + + static create() { + const messageId = new MessageId(); + messageId.version = MESSAGE_VERSION.V1; + messageId.macAddress = this.MAC; + messageId.processId = this.#processId; + messageId.timestamp = this.#getCurrentTimestamp(); + messageId.sequence = this.#sequence++; + if (this.#sequence > MAX_UINT32) { + this.#sequence = 0; + } + this.#buf.writeUInt8(messageId.version, 0); + this.#buf.write(messageId.macAddress, 1, 'hex'); + this.#buf.writeUInt16BE(messageId.processId, 7); + this.#buf.writeUInt32BE(messageId.timestamp, 9); + this.#buf.writeUInt32BE(messageId.sequence, 13); + messageId.id = this.#buf.toString('hex').toUpperCase(); + return messageId; + } + + static decode(id: string) { + const messageId = new MessageId(); + messageId.id = id; + this.#buf.write(id, 0, 'hex'); + messageId.version = this.#buf.readUInt8(0); + messageId.macAddress = this.#buf.subarray(1, 7).toString('hex'); + messageId.processId = this.#buf.readUInt16BE(7); + messageId.timestamp = this.#buf.readUInt32BE(9); + messageId.sequence = this.#buf.readUInt32BE(13); + return messageId; + } + + static #getCurrentTimestamp() { + // use lower 4 bytes + return Math.floor(Date.now() / 1000 - this.#sinceTimestamp) % MAX_UINT32; + } +} + +// set current mac address +address.mac((err, mac) => { + if (err) { + console.warn('[rocketmq-client-nodejs] can\'t get mac address, %s', err.message); + return; + } + if (!mac) { + console.warn('[rocketmq-client-nodejs] can\'t get mac address'); + return; + } + MessageIdFactory.MAC = mac.replaceAll(':', ''); +}); diff --git a/nodejs/src/message/MessageView.ts b/nodejs/src/message/MessageView.ts new file mode 100644 index 000000000..d2aad7381 --- /dev/null +++ b/nodejs/src/message/MessageView.ts @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { gunzipSync } from 'node:zlib'; +import { + DigestType, + Encoding, + Message as MessagePB, +} from '../../proto/apache/rocketmq/v2/definition_pb'; +import { crc32CheckSum, md5CheckSum, sha1CheckSum } from '../util'; +import { Endpoints, MessageQueue } from '../route'; + +export class MessageView { + readonly messageId: string; + readonly topic: string; + readonly body: Buffer; + readonly corrupted: boolean; + readonly transportDeliveryTimestamp?: Date; + readonly tag?: string; + readonly messageGroup?: string; + readonly deliveryTimestamp?: Date; + readonly keys: string[]; + readonly bornHost: string; + readonly bornTimestamp?: Date; + readonly deliveryAttempt?: number; + readonly endpoints: Endpoints; + readonly receiptHandle: string; + readonly offset?: number; + readonly decodeTimestamp: Date; + readonly properties = new Map(); + + constructor(message: MessagePB, messageQueue?: MessageQueue, transportDeliveryTimestamp?: Date) { + const systemProperties = message.getSystemProperties()!; + const bodyDigest = systemProperties.getBodyDigest()!.toObject(); + const digestType = bodyDigest.type; + const checksum = bodyDigest.checksum; + let expectedChecksum = ''; + let bodyBytes = Buffer.from(message.getBody_asU8()); + switch (digestType) { + case DigestType.CRC32: + expectedChecksum = crc32CheckSum(bodyBytes); + break; + case DigestType.MD5: + expectedChecksum = md5CheckSum(bodyBytes); + break; + case DigestType.SHA1: + expectedChecksum = sha1CheckSum(bodyBytes); + break; + default: + // log.error("Unsupported message body digest algorithm, digestType={}, topic={}, messageId={}", + // digestType, topic, messageId); + } + if (expectedChecksum && expectedChecksum !== checksum) { + this.corrupted = true; + } + if (systemProperties.getBodyEncoding() === Encoding.GZIP) { + bodyBytes = gunzipSync(bodyBytes); + } + + for (const [ key, value ] of message.getUserPropertiesMap().entries()) { + this.properties.set(key, value); + } + this.messageId = systemProperties.getMessageId(); + this.topic = message.getTopic()!.getName(); + this.tag = systemProperties.getTag(); + this.messageGroup = systemProperties.getMessageGroup(); + this.deliveryTimestamp = systemProperties.getDeliveryTimestamp()?.toDate(); + this.keys = systemProperties.getKeysList(); + this.bornHost = systemProperties.getBornHost(); + this.bornTimestamp = systemProperties.getBornTimestamp()?.toDate(); + this.deliveryAttempt = systemProperties.getDeliveryAttempt(); + this.offset = systemProperties.getQueueOffset(); + this.receiptHandle = systemProperties.getReceiptHandle()!; + this.transportDeliveryTimestamp = transportDeliveryTimestamp; + if (messageQueue) { + this.endpoints = messageQueue.broker.endpoints; + } + this.body = bodyBytes; + } +} diff --git a/nodejs/src/message/PublishingMessage.ts b/nodejs/src/message/PublishingMessage.ts new file mode 100644 index 000000000..b6015bf37 --- /dev/null +++ b/nodejs/src/message/PublishingMessage.ts @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Timestamp } from 'google-protobuf/google/protobuf/timestamp_pb'; +import { + MessageType, Message as MessagePB, SystemProperties, Encoding, +} from '../../proto/apache/rocketmq/v2/definition_pb'; +import { PublishingSettings } from '../producer'; +import { createResource } from '../util'; +import { MessageQueue } from '../route'; +import { UserAgent } from '../client'; +import { Message, MessageOptions } from './Message'; +import { MessageIdFactory } from './MessageId'; + +export class PublishingMessage extends Message { + readonly messageId: string; + readonly messageType: MessageType; + + constructor(options: MessageOptions, publishingSettings: PublishingSettings, txEnabled: boolean) { + super(options); + const length = this.body.length; + const maxBodySizeBytes = publishingSettings.maxBodySizeBytes; + if (length > maxBodySizeBytes) { + throw new TypeError(`Message body size exceeds the threshold, max size=${maxBodySizeBytes} bytes`); + } + // Generate message id. + this.messageId = MessageIdFactory.create().toString(); + // Normal message. + if (!this.messageGroup && !this.deliveryTimestamp && !txEnabled) { + this.messageType = MessageType.NORMAL; + return; + } + // Fifo message. + if (this.messageGroup && !txEnabled) { + this.messageType = MessageType.FIFO; + return; + } + // Delay message. + if (this.deliveryTimestamp && !txEnabled) { + this.messageType = MessageType.DELAY; + return; + } + // Transaction message. + if (!this.messageGroup && + !this.deliveryTimestamp && txEnabled) { + this.messageType = MessageType.TRANSACTION; + return; + } + // Transaction semantics is conflicted with fifo/delay. + throw new TypeError('Transactional message should not set messageGroup or deliveryTimestamp'); + } + + /** + * This method should be invoked before each message sending, because the born time is reset before each + * invocation, which means that it should not be invoked ahead of time. + */ + toProtobuf(mq: MessageQueue) { + const systemProperties = new SystemProperties() + .setKeysList(this.keys) + .setMessageId(this.messageId) + .setBornTimestamp(Timestamp.fromDate(new Date())) + .setBornHost(UserAgent.INSTANCE.hostname) + .setBodyEncoding(Encoding.IDENTITY) + .setQueueId(mq.queueId) + .setMessageType(this.messageType); + if (this.tag) { + systemProperties.setTag(this.tag); + } + if (this.deliveryTimestamp) { + systemProperties.setDeliveryTimestamp(Timestamp.fromDate(this.deliveryTimestamp)); + } + if (this.messageGroup) { + systemProperties.setMessageGroup(this.messageGroup); + } + + const message = new MessagePB() + .setTopic(createResource(this.topic)) + .setBody(this.body) + .setSystemProperties(systemProperties); + if (this.properties) { + const userProperties = message.getUserPropertiesMap(); + for (const [ key, value ] of this.properties.entries()) { + userProperties.set(key, value); + } + } + return message; + } +} diff --git a/nodejs/src/message/index.ts b/nodejs/src/message/index.ts new file mode 100644 index 000000000..a805d0edc --- /dev/null +++ b/nodejs/src/message/index.ts @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './Message'; +export * from './MessageId'; +export * from './MessageView'; +export * from './PublishingMessage'; diff --git a/nodejs/src/producer/Producer.ts b/nodejs/src/producer/Producer.ts new file mode 100644 index 000000000..623ecf461 --- /dev/null +++ b/nodejs/src/producer/Producer.ts @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'node:assert'; +import { setTimeout } from 'node:timers/promises'; +import { + ClientType, + MessageType, + TransactionResolution, +} from '../../proto/apache/rocketmq/v2/definition_pb'; +import { + EndTransactionRequest, + HeartbeatRequest, + NotifyClientTerminationRequest, + RecoverOrphanedTransactionCommand, + SendMessageRequest, +} from '../../proto/apache/rocketmq/v2/service_pb'; +import { + Endpoints, + MessageQueue, + TopicRouteData, +} from '../route'; +import { + ExponentialBackoffRetryPolicy, +} from '../retry'; +import { StatusChecker, TooManyRequestsException } from '../exception'; +import { BaseClient, BaseClientOptions, Settings } from '../client'; +import { PublishingMessage, MessageOptions, MessageView, Message } from '../message'; +import { PublishingSettings } from './PublishingSettings'; +import { TransactionChecker } from './TransactionChecker'; +import { PublishingLoadBalancer } from './PublishingLoadBalancer'; +import { SendReceipt } from './SendReceipt'; +import { Transaction } from './Transaction'; +import { createResource } from '../util'; + +export interface ProducerOptions extends BaseClientOptions { + topic?: string | string[]; + maxAttempts?: number; + checker?: TransactionChecker; +} + +export class Producer extends BaseClient { + #publishingSettings: PublishingSettings; + #checker?: TransactionChecker; + #publishingRouteDataCache = new Map(); + + constructor(options: ProducerOptions) { + if (!options.topics && options.topic) { + options.topics = Array.isArray(options.topic) ? options.topic : [ options.topic ]; + } + super(options); + // https://rocketmq.apache.org/docs/introduction/03limits/ + // Default max number of message sending retries is 3 + const retryPolicy = ExponentialBackoffRetryPolicy.immediatelyRetryPolicy(options.maxAttempts ?? 3); + this.#publishingSettings = new PublishingSettings(this.clientId, this.endpoints, retryPolicy, + this.requestTimeout, this.topics); + this.#checker = options.checker; + } + + get publishingSettings() { + return this.#publishingSettings; + } + + beginTransaction() { + assert(this.#checker, 'Transaction checker should not be null'); + return new Transaction(this); + } + + async endTransaction(endpoints: Endpoints, message: Message, messageId: string, + transactionId: string, resolution: TransactionResolution) { + const request = new EndTransactionRequest() + .setMessageId(messageId) + .setTransactionId(transactionId) + .setTopic(createResource(message.topic)) + .setResolution(resolution); + const response = await this.rpcClientManager.endTransaction(endpoints, request, this.requestTimeout); + StatusChecker.check(response.getStatus()?.toObject()); + } + + async onRecoverOrphanedTransactionCommand(endpoints: Endpoints, command: RecoverOrphanedTransactionCommand) { + const transactionId = command.getTransactionId(); + const messagePB = command.getMessage()!; + const messageId = messagePB.getSystemProperties()!.getMessageId(); + if (!this.#checker) { + this.logger.error('No transaction checker registered, ignore it, messageId=%s, transactionId=%s, endpoints=%s, clientId=%s', + messageId, transactionId, endpoints, this.clientId); + return; + } + let messageView: MessageView; + try { + messageView = new MessageView(messagePB); + } catch (err) { + this.logger.error('[Bug] Failed to decode message during orphaned transaction message recovery, messageId=%s, transactionId=%s, endpoints=%s, clientId=%s, error=%s', + messageId, transactionId, endpoints, this.clientId, err); + return; + } + + try { + const resolution = await this.#checker.check(messageView); + if (resolution === null || resolution === TransactionResolution.TRANSACTION_RESOLUTION_UNSPECIFIED) { + return; + } + await this.endTransaction(endpoints, messageView, messageId, transactionId, resolution); + this.logger.info('Recover orphaned transaction message success, transactionId=%s, resolution=%s, messageId=%s, clientId=%s', + transactionId, resolution, messageId, this.clientId); + } catch (err) { + this.logger.error('Exception raised while checking the transaction, messageId=%s, transactionId=%s, endpoints=%s, clientId=%s, error=%s', + messageId, transactionId, endpoints, this.clientId, err); + return; + } + } + + protected getSettings(): Settings { + return this.#publishingSettings; + } + + protected wrapHeartbeatRequest(): HeartbeatRequest { + return new HeartbeatRequest() + .setClientType(ClientType.PRODUCER); + } + + protected wrapNotifyClientTerminationRequest(): NotifyClientTerminationRequest { + return new NotifyClientTerminationRequest(); + } + + async send(message: MessageOptions, transaction?: Transaction) { + if (!transaction) { + const sendReceipts = await this.#send([ message ], false); + return sendReceipts[0]; + } + + const publishingMessage = transaction.tryAddMessage(message); + const sendReceipts = await this.#send([ message ], true); + const sendReceipt = sendReceipts[0]; + transaction.tryAddReceipt(publishingMessage, sendReceipt); + return sendReceipt; + } + + async #send(messages: MessageOptions[], txEnabled: boolean) { + const pubMessages: PublishingMessage[] = []; + const topics = new Set(); + for (const message of messages) { + pubMessages.push(new PublishingMessage(message, this.#publishingSettings, txEnabled)); + topics.add(message.topic); + } + if (topics.size > 1) { + throw new TypeError(`Messages to send have different topics=${JSON.stringify(topics)}`); + } + const topic = pubMessages[0].topic; + const messageType = pubMessages[0].messageType; + const messageGroup = pubMessages[0].messageGroup; + const messageTypes = new Set(pubMessages.map(m => m.messageType)); + if (messageTypes.size > 1) { + throw new TypeError(`Messages to send have different types=${JSON.stringify(messageTypes)}`); + } + + // Message group must be same if message type is FIFO, or no need to proceed. + if (messageType === MessageType.FIFO) { + const messageGroups = new Set(pubMessages.map(m => m.messageGroup!)); + if (messageGroups.size > 1) { + throw new TypeError(`FIFO messages to send have message groups, messageGroups=${JSON.stringify(messageGroups)}`); + } + } + + // Get publishing topic route. + const loadBalancer = await this.#getPublishingLoadBalancer(topic); + // Prepare the candidate message queue(s) for retry-sending in advance. + const candidates = messageGroup ? [ loadBalancer.takeMessageQueueByMessageGroup(messageGroup) ] : + this.#takeMessageQueues(loadBalancer); + return await this.#send0(topic, messageType, candidates, pubMessages, 1); + } + + #wrapSendMessageRequest(pubMessages: PublishingMessage[], mq: MessageQueue) { + const request = new SendMessageRequest(); + for (const pubMessage of pubMessages) { + request.addMessages(pubMessage.toProtobuf(mq)); + } + return request; + } + + /** + * Isolate specified Endpoints + */ + #isolate(endpoints: Endpoints) { + this.isolated.set(endpoints.facade, endpoints); + } + + async #send0(topic: string, messageType: MessageType, candidates: MessageQueue[], + messages: PublishingMessage[], attempt: number): Promise { + // Calculate the current message queue. + const index = (attempt - 1) % candidates.length; + const mq = candidates[index]; + const acceptMessageTypes = mq.acceptMessageTypesList; + if (this.#publishingSettings.isValidateMessageType() && !acceptMessageTypes.includes(messageType)) { + throw new TypeError('Current message type not match with ' + + 'topic accept message types, topic=' + topic + ', actualMessageType=' + messageType + ', ' + + 'acceptMessageTypes=' + JSON.stringify(acceptMessageTypes)); + } + const endpoints = mq.broker.endpoints; + const maxAttempts = this.#getRetryPolicy().getMaxAttempts(); + const request = this.#wrapSendMessageRequest(messages, mq); + let sendReceipts: SendReceipt[] = []; + try { + const response = await this.rpcClientManager.sendMessage(endpoints, request, this.requestTimeout); + sendReceipts = SendReceipt.processResponseInvocation(mq, response); + } catch (err) { + const messageIds = messages.map(m => m.messageId); + // Isolate endpoints because of sending failure. + this.#isolate(endpoints); + if (attempt >= maxAttempts) { + // No need more attempts. + this.logger.error('Failed to send message(s) finally, run out of attempt times, maxAttempts=%s, attempt=%s, topic=%s, messageId(s)=%s, endpoints=%s, clientId=%s, error=%s', + maxAttempts, attempt, topic, messageIds, endpoints, this.clientId, err); + throw err; + } + // No need more attempts for transactional message. + if (messageType === MessageType.TRANSACTION) { + this.logger.error('Failed to send transactional message finally, maxAttempts=%s, attempt=%s, topic=%s, messageId(s)=%s, endpoints=%s, clientId=%s, error=%s', + maxAttempts, attempt, topic, messageIds, endpoints, this.clientId, err); + throw err; + } + // Try to do more attempts. + const nextAttempt = 1 + attempt; + // Retry immediately if the request is not throttled. + if (!(err instanceof TooManyRequestsException)) { + this.logger.warn('Failed to send message, would attempt to resend right now, maxAttempts=%s, attempt=%s, topic=%s, messageId(s)=%s, endpoints=%s, clientId=%s, error=%s', + maxAttempts, attempt, topic, messageIds, endpoints, this.clientId, err); + return this.#send0(topic, messageType, candidates, messages, nextAttempt); + } + const delay = this.#getRetryPolicy().getNextAttemptDelay(nextAttempt); + this.logger.warn('Failed to send message due to too many requests, would attempt to resend after %sms, maxAttempts=%s, attempt=%s, topic=%s, messageId(s)=%s, endpoints=%s, clientId=%s, error=%s', + delay, maxAttempts, attempt, topic, messageIds, endpoints, this.clientId, err); + await setTimeout(delay); + return this.#send0(topic, messageType, candidates, messages, nextAttempt); + } + + // Resend message(s) successfully. + if (attempt > 1) { + const messageIds = sendReceipts.map(r => r.messageId); + this.logger.info('Resend message successfully, topic=%s, messageId(s)=%j, maxAttempts=%s, attempt=%s, endpoints=%s, clientId=%s', + topic, messageIds, maxAttempts, attempt, endpoints, this.clientId); + } + // Send message(s) successfully on first attempt, return directly. + return sendReceipts; + } + + async #getPublishingLoadBalancer(topic: string) { + let loadBalancer = this.#publishingRouteDataCache.get(topic); + if (!loadBalancer) { + const topicRouteData = await this.getRouteData(topic); + loadBalancer = this.#updatePublishingLoadBalancer(topic, topicRouteData); + } + return loadBalancer; + } + + #updatePublishingLoadBalancer(topic: string, topicRouteData: TopicRouteData) { + let loadBalancer = this.#publishingRouteDataCache.get(topic); + if (loadBalancer) { + loadBalancer = loadBalancer.update(topicRouteData); + } else { + loadBalancer = new PublishingLoadBalancer(topicRouteData); + } + this.#publishingRouteDataCache.set(topic, loadBalancer); + return loadBalancer; + } + + /** + * Take message queue(s) from route for message publishing. + */ + #takeMessageQueues(loadBalancer: PublishingLoadBalancer) { + return loadBalancer.takeMessageQueues(this.isolated, this.#getRetryPolicy().getMaxAttempts()); + } + + #getRetryPolicy() { + return this.#publishingSettings.getRetryPolicy()!; + } +} diff --git a/nodejs/src/producer/PublishingLoadBalancer.ts b/nodejs/src/producer/PublishingLoadBalancer.ts new file mode 100644 index 000000000..b04300ffc --- /dev/null +++ b/nodejs/src/producer/PublishingLoadBalancer.ts @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { randomInt } from 'node:crypto'; +import { Permission } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { Endpoints, MessageQueue, TopicRouteData } from '../route'; +import { MASTER_BROKER_ID, calculateStringSipHash24 } from '../util'; + +export class PublishingLoadBalancer { + #index: number; + #messageQueues: MessageQueue[]; + + constructor(topicRouteData: TopicRouteData, index?: number) { + this.#messageQueues = topicRouteData.messageQueues.filter(mq => { + return mq.queueId === MASTER_BROKER_ID && (mq.permission === Permission.WRITE || mq.permission === Permission.READ_WRITE); + }); + this.#index = index === undefined ? randomInt(this.#messageQueues.length) : index; + if (this.#messageQueues.length === 0) { + throw new Error(`No writable message queue found, topicRouteData=${JSON.stringify(topicRouteData)}`); + } + } + + update(topicRouteData: TopicRouteData) { + return new PublishingLoadBalancer(topicRouteData, this.#index); + } + + takeMessageQueues(excluded: Map, count: number) { + if (this.#index >= this.#messageQueues.length) { + this.#index = 0; + } + let next = this.#index++; + const candidates: MessageQueue[] = []; + const candidateBrokerNames = new Set(); + + const size = this.#messageQueues.length; + for (let i = 0; i < size; i++) { + const messageQueue = this.#messageQueues[next++ % size]; + const broker = messageQueue.broker; + const brokerName = broker.name; + if (!excluded.has(broker.endpoints.facade) && !candidateBrokerNames.has(brokerName)) { + candidateBrokerNames.add(brokerName); + candidates.push(messageQueue); + } + if (candidates.length >= count) { + return candidates; + } + } + // If all endpoints are isolated. + if (candidates.length === 0) { + for (let i = 0; i < size; i++) { + const messageQueue = this.#messageQueues[next++ % size]; + const broker = messageQueue.broker; + const brokerName = broker.name; + if (!candidateBrokerNames.has(brokerName)) { + candidateBrokerNames.add(brokerName); + candidates.push(messageQueue); + } + if (candidates.length >= count) { + return candidates; + } + } + } + return candidates; + } + + takeMessageQueueByMessageGroup(messageGroup: string) { + const hashCode = calculateStringSipHash24(messageGroup); + const index = parseInt(`${hashCode % BigInt(this.#messageQueues.length)}`); + return this.#messageQueues[index]; + } +} diff --git a/nodejs/src/producer/PublishingSettings.ts b/nodejs/src/producer/PublishingSettings.ts new file mode 100644 index 000000000..fb72d66dd --- /dev/null +++ b/nodejs/src/producer/PublishingSettings.ts @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Settings as SettingsPB, + ClientType, + Publishing, +} from '../../proto/apache/rocketmq/v2/definition_pb'; +import { Endpoints } from '../route'; +import { ExponentialBackoffRetryPolicy } from '../retry'; +import { Settings, UserAgent } from '../client'; +import { createDuration } from '../util'; + +export class PublishingSettings extends Settings { + readonly #topics: Set; + /** + * If message body size exceeds the threshold, it would be compressed for convenience of transport. + * https://rocketmq.apache.org/docs/introduction/03limits/ + * Default max message size is 4 MB + */ + #maxBodySizeBytes = 4 * 1024 * 1024; + #validateMessageType = true; + + constructor(clientId: string, accessPoint: Endpoints, retryPolicy: ExponentialBackoffRetryPolicy, + requestTimeout: number, topics: Set) { + super(clientId, ClientType.PRODUCER, accessPoint, requestTimeout, retryPolicy); + this.#topics = topics; + } + + get maxBodySizeBytes() { + return this.#maxBodySizeBytes; + } + + isValidateMessageType() { + return this.#validateMessageType; + } + + toProtobuf(): SettingsPB { + const publishing = new Publishing() + .setValidateMessageType(this.#validateMessageType); + for (const topic of this.#topics) { + publishing.addTopics().setName(topic); + } + return new SettingsPB() + .setClientType(this.clientType) + .setAccessPoint(this.accessPoint.toProtobuf()) + .setRequestTimeout(createDuration(this.requestTimeout)) + .setPublishing(publishing) + .setUserAgent(UserAgent.INSTANCE.toProtobuf()); + } + + sync(settings: SettingsPB): void { + if (settings.getPubSubCase() !== SettingsPB.PubSubCase.PUBLISHING) { + // log.error("[Bug] Issued settings not match with the client type, clientId={}, pubSubCase={}, " + // + "clientType={}", clientId, pubSubCase, clientType); + return; + } + const backoffPolicy = settings.getBackoffPolicy()!; + const publishing = settings.getPublishing()!.toObject(); + const exist = this.retryPolicy!; + this.retryPolicy = exist.inheritBackoff(backoffPolicy); + this.#validateMessageType = publishing.validateMessageType; + this.#maxBodySizeBytes = publishing.maxBodySize; + } +} diff --git a/nodejs/src/producer/SendReceipt.ts b/nodejs/src/producer/SendReceipt.ts new file mode 100644 index 000000000..0403aea7a --- /dev/null +++ b/nodejs/src/producer/SendReceipt.ts @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Code } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { SendMessageResponse } from '../../proto/apache/rocketmq/v2/service_pb'; +import { MessageQueue } from '../route'; +import { StatusChecker } from '../exception'; + +export class SendReceipt { + readonly messageId: string; + readonly transactionId: string; + readonly offset: number; + readonly #messageQueue: MessageQueue; + + constructor(messageId: string, transactionId: string, messageQueue: MessageQueue, offset: number) { + this.messageId = messageId; + this.transactionId = transactionId; + this.offset = offset; + this.#messageQueue = messageQueue; + } + + get messageQueue() { + return this.#messageQueue; + } + + get endpoints() { + return this.#messageQueue.broker.endpoints; + } + + static processResponseInvocation(mq: MessageQueue, response: SendMessageResponse) { + const responseObj = response.toObject(); + // Filter abnormal status. + const abnormalStatus = responseObj.entriesList.map(e => e.status).find(s => s?.code !== Code.OK); + const status = abnormalStatus ?? responseObj.status; + StatusChecker.check(status); + const sendReceipts: SendReceipt[] = []; + for (const entry of responseObj.entriesList) { + const messageId = entry.messageId; + const transactionId = entry.transactionId; + const offset = entry.offset; + const sendReceipt = new SendReceipt(messageId, transactionId, mq, offset); + sendReceipts.push(sendReceipt); + } + return sendReceipts; + } +} diff --git a/nodejs/src/producer/Transaction.ts b/nodejs/src/producer/Transaction.ts new file mode 100644 index 000000000..85d525ea8 --- /dev/null +++ b/nodejs/src/producer/Transaction.ts @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TransactionResolution } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { MessageOptions, PublishingMessage } from '../message'; +import type { Producer } from './Producer'; +import { SendReceipt } from './SendReceipt'; + +export class Transaction { + static readonly MAX_MESSAGE_NUM = 1; + readonly #producer: Producer; + readonly #messageMap = new Map(); + readonly #messageSendReceiptMap = new Map(); + + constructor(producer: Producer) { + this.#producer = producer; + } + + tryAddMessage(message: MessageOptions) { + if (this.#messageMap.size >= Transaction.MAX_MESSAGE_NUM) { + throw new TypeError(`Message in transaction has exceeded the threshold=${Transaction.MAX_MESSAGE_NUM}`); + } + const publishingMessage = new PublishingMessage(message, this.#producer.publishingSettings, true); + this.#messageMap.set(publishingMessage.messageId, publishingMessage); + return publishingMessage; + } + + tryAddReceipt(publishingMessage: PublishingMessage, sendReceipt: SendReceipt) { + if (!this.#messageMap.has(publishingMessage.messageId)) { + throw new TypeError('Message not in transaction'); + } + this.#messageSendReceiptMap.set(publishingMessage.messageId, sendReceipt); + } + + async commit() { + if (this.#messageSendReceiptMap.size === 0) { + throw new TypeError('Transactional message has not been sent yet'); + } + for (const [ messageId, sendReceipt ] of this.#messageSendReceiptMap.entries()) { + const publishingMessage = this.#messageMap.get(messageId)!; + await this.#producer.endTransaction(sendReceipt.endpoints, publishingMessage, + sendReceipt.messageId, sendReceipt.transactionId, TransactionResolution.COMMIT); + } + } + + async rollback() { + if (this.#messageSendReceiptMap.size === 0) { + throw new TypeError('Transactional message has not been sent yet'); + } + for (const [ messageId, sendReceipt ] of this.#messageSendReceiptMap.entries()) { + const publishingMessage = this.#messageMap.get(messageId)!; + await this.#producer.endTransaction(sendReceipt.endpoints, publishingMessage, + sendReceipt.messageId, sendReceipt.transactionId, TransactionResolution.ROLLBACK); + } + } +} diff --git a/nodejs/src/producer/TransactionChecker.ts b/nodejs/src/producer/TransactionChecker.ts new file mode 100644 index 000000000..759492dc1 --- /dev/null +++ b/nodejs/src/producer/TransactionChecker.ts @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TransactionResolution } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { MessageView } from '../message'; + +export interface TransactionChecker { + check(messageView: MessageView): Promise; +} diff --git a/nodejs/src/producer/index.ts b/nodejs/src/producer/index.ts new file mode 100644 index 000000000..1380581f2 --- /dev/null +++ b/nodejs/src/producer/index.ts @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './Producer'; +export * from './PublishingLoadBalancer'; +export * from './PublishingSettings'; +export * from './SendReceipt'; +// export * from './Transaction'; +export * from './TransactionChecker'; diff --git a/nodejs/src/retry/ExponentialBackoffRetryPolicy.ts b/nodejs/src/retry/ExponentialBackoffRetryPolicy.ts new file mode 100644 index 000000000..3fc8997b9 --- /dev/null +++ b/nodejs/src/retry/ExponentialBackoffRetryPolicy.ts @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'node:assert'; +import { Duration } from 'google-protobuf/google/protobuf/duration_pb'; +import { + RetryPolicy as RetryPolicyPB, + ExponentialBackoff, +} from '../../proto/apache/rocketmq/v2/definition_pb'; +import { RetryPolicy } from './RetryPolicy'; + +export class ExponentialBackoffRetryPolicy implements RetryPolicy { + #maxAttempts: number; + // seconds + #initialBackoff: number; + #maxBackoff: number; + #backoffMultiplier: number; + + constructor(maxAttempts: number, initialBackoff = 0, maxBackoff = 0, backoffMultiplier = 1) { + this.#maxAttempts = maxAttempts; + this.#initialBackoff = initialBackoff; + this.#maxBackoff = maxBackoff; + this.#backoffMultiplier = backoffMultiplier; + } + + static immediatelyRetryPolicy(maxAttempts: number) { + return new ExponentialBackoffRetryPolicy(maxAttempts, 0, 0, 1); + } + + getMaxAttempts(): number { + return this.#maxAttempts; + } + + getNextAttemptDelay(attempt: number): number { + assert(attempt > 0, 'attempt must be positive'); + const delay = Math.min(this.#initialBackoff * Math.pow(this.#backoffMultiplier, 1.0 * (attempt - 1)), this.#maxBackoff); + if (delay <= 0) { + return 0; + } + return delay; + } + + inheritBackoff(retryPolicy: RetryPolicyPB): RetryPolicy { + assert(retryPolicy.getStrategyCase() === RetryPolicyPB.StrategyCase.EXPONENTIAL_BACKOFF, + 'strategy must be exponential backoff'); + const backoff = retryPolicy.getExponentialBackoff()!.toObject(); + return new ExponentialBackoffRetryPolicy(this.#maxAttempts, + backoff.initial?.seconds, + backoff.max?.seconds, + backoff.multiplier); + } + + toProtobuf(): RetryPolicyPB { + return new RetryPolicyPB() + .setMaxAttempts(this.#maxAttempts) + .setExponentialBackoff( + new ExponentialBackoff() + .setInitial(new Duration().setSeconds(this.#initialBackoff)) + .setMax(new Duration().setSeconds(this.#maxBackoff)) + .setMultiplier(this.#backoffMultiplier)); + } +} diff --git a/nodejs/src/retry/RetryPolicy.ts b/nodejs/src/retry/RetryPolicy.ts new file mode 100644 index 000000000..0e559d348 --- /dev/null +++ b/nodejs/src/retry/RetryPolicy.ts @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RetryPolicy as RetryPolicyPB } from '../../proto/apache/rocketmq/v2/definition_pb'; + +/** + * Internal interface for retry policy. + */ +export interface RetryPolicy { + /** + * Get the max attempt times for retry. + * + * @return max attempt times. + */ + getMaxAttempts(): number; + + /** + * Get await time after current attempts, the attempt index starts at 1. + * + * @param attempt current attempt. + * @return await time in seconds. + */ + getNextAttemptDelay(attempt: number): number; + + /** + * Update the retry backoff strategy and generate a new one. + * + * @param retryPolicy retry policy which contains the backoff strategy. + * @return the new retry policy. + */ + inheritBackoff(retryPolicy: RetryPolicyPB): RetryPolicy; + + /** + * Convert to RetryPolicyPB + */ + toProtobuf(): RetryPolicyPB; +} diff --git a/nodejs/src/retry/index.ts b/nodejs/src/retry/index.ts new file mode 100644 index 000000000..79d29d0f2 --- /dev/null +++ b/nodejs/src/retry/index.ts @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './ExponentialBackoffRetryPolicy'; +export * from './RetryPolicy'; diff --git a/nodejs/src/route/Broker.ts b/nodejs/src/route/Broker.ts new file mode 100644 index 000000000..3923b7a72 --- /dev/null +++ b/nodejs/src/route/Broker.ts @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Broker as BrokerPB } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { Endpoints } from './Endpoints'; + +export class Broker implements BrokerPB.AsObject { + name: string; + id: number; + endpoints: Endpoints; + + constructor(broker: BrokerPB.AsObject) { + this.name = broker.name; + this.id = broker.id; + this.endpoints = new Endpoints(broker.endpoints!); + } + + toProtobuf() { + const broker = new BrokerPB(); + broker.setName(this.name); + broker.setId(this.id); + broker.setEndpoints(this.endpoints.toProtobuf()); + return broker; + } +} diff --git a/nodejs/src/route/Endpoints.ts b/nodejs/src/route/Endpoints.ts new file mode 100644 index 000000000..6ac7332fe --- /dev/null +++ b/nodejs/src/route/Endpoints.ts @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isIPv4, isIPv6 } from 'node:net'; +import { Address, AddressScheme, Endpoints as EndpointsPB } from '../../proto/apache/rocketmq/v2/definition_pb'; + +const DEFAULT_PORT = 80; + +export class Endpoints { + readonly addressesList: Address.AsObject[]; + readonly scheme: AddressScheme; + /** + * URI path for grpc target, e.g: + * 127.0.0.1:10911[,127.0.0.2:10912] + */ + readonly facade: string; + + constructor(endpoints: string | EndpointsPB.AsObject) { + if (typeof endpoints === 'string') { + const splits = endpoints.split(';'); + this.addressesList = []; + for (const endpoint of splits) { + const [ host, port ] = endpoint.split(':'); + if (isIPv4(host)) { + this.scheme = AddressScheme.IPV4; + } else if (isIPv6(host)) { + this.scheme = AddressScheme.IPV6; + } else { + this.scheme = AddressScheme.DOMAIN_NAME; + } + this.addressesList.push({ host, port: parseInt(port) || DEFAULT_PORT }); + } + } else { + this.scheme = endpoints.scheme; + this.addressesList = endpoints.addressesList; + } + this.facade = this.addressesList.map(addr => `${addr.host}:${addr.port}`).join(','); + } + + getGrpcTarget() { + return this.facade; + } + + toString() { + return this.facade; + } + + toProtobuf() { + const endpoints = new EndpointsPB(); + endpoints.setScheme(this.scheme); + for (const address of this.addressesList) { + endpoints.addAddresses().setHost(address.host).setPort(address.port); + } + return endpoints; + } +} diff --git a/nodejs/src/route/MessageQueue.ts b/nodejs/src/route/MessageQueue.ts new file mode 100644 index 000000000..1569b75ea --- /dev/null +++ b/nodejs/src/route/MessageQueue.ts @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + MessageQueue as MessageQueuePB, MessageType, Permission, + Resource, +} from '../../proto/apache/rocketmq/v2/definition_pb'; +import { createResource } from '../util'; +import { Broker } from './Broker'; + +export class MessageQueue { + topic: Resource.AsObject; + queueId: number; + broker: Broker; + permission: Permission; + acceptMessageTypesList: MessageType[]; + + constructor(messageQueue: MessageQueuePB) { + this.topic = messageQueue.getTopic()!.toObject(); + this.queueId = messageQueue.getId(); + this.permission = messageQueue.getPermission(); + this.acceptMessageTypesList = messageQueue.getAcceptMessageTypesList(); + this.broker = new Broker(messageQueue.getBroker()!.toObject()); + } + + toProtobuf() { + const messageQueue = new MessageQueuePB(); + messageQueue.setId(this.queueId); + messageQueue.setTopic(createResource(this.topic.name)); + messageQueue.setBroker(this.broker.toProtobuf()); + messageQueue.setPermission(this.permission); + messageQueue.setAcceptMessageTypesList(this.acceptMessageTypesList); + return messageQueue; + } +} diff --git a/nodejs/src/route/TopicRouteData.ts b/nodejs/src/route/TopicRouteData.ts new file mode 100644 index 000000000..6abae444a --- /dev/null +++ b/nodejs/src/route/TopicRouteData.ts @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MessageQueue as MessageQueuePB } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { Endpoints } from './Endpoints'; +import { MessageQueue } from './MessageQueue'; + +export class TopicRouteData { + readonly messageQueues: MessageQueue[] = []; + + constructor(messageQueues: MessageQueuePB[]) { + for (const mq of messageQueues) { + this.messageQueues.push(new MessageQueue(mq)); + } + } + + getTotalEndpoints() { + const endpointsMap = new Map(); + for (const mq of this.messageQueues) { + endpointsMap.set(mq.broker.endpoints.facade, mq.broker.endpoints); + } + return Array.from(endpointsMap.values()); + } +} diff --git a/nodejs/src/route/index.ts b/nodejs/src/route/index.ts new file mode 100644 index 000000000..571e4f92c --- /dev/null +++ b/nodejs/src/route/index.ts @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './Broker'; +export * from './Endpoints'; +export * from './MessageQueue'; +export * from './TopicRouteData'; diff --git a/nodejs/src/util/index.ts b/nodejs/src/util/index.ts new file mode 100644 index 000000000..7b4243fef --- /dev/null +++ b/nodejs/src/util/index.ts @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { performance } from 'node:perf_hooks'; +import { createHash, createHmac } from 'node:crypto'; +import { Duration } from 'google-protobuf/google/protobuf/duration_pb'; +import { crc32 } from '@node-rs/crc32'; +import siphash24 from 'siphash24'; +import { Resource } from '../../proto/apache/rocketmq/v2/definition_pb'; + +export const MASTER_BROKER_ID = 0; + +export function getTimestamp() { + const timestamp = performance.timeOrigin + performance.now(); + const seconds = Math.floor(timestamp / 1000); + const nanos = Math.floor((timestamp % 1000) * 1e6); + return { seconds, nanos, timestamp }; +} + +// DATE_TIME_FORMAT = "yyyyMMdd'T'HHmmss'Z'" +export function getRequestDateTime() { + // 2023-09-13T06:30:59.399Z => 20230913T063059Z + const now = new Date().toISOString().split('.')[0].replace(/[\-\:]/g, ''); + return `${now}Z`; +} + +export function sign(accessSecret: string, dateTime: string) { + const hmacSha1 = createHmac('sha1', accessSecret); + hmacSha1.update(dateTime); + return hmacSha1.digest('hex').toUpperCase(); +} + +export function createDuration(ms: number) { + const nanos = ms % 1000 * 1000000; + return new Duration() + .setSeconds(ms / 1000) + .setNanos(nanos); +} + +export function createResource(name: string) { + return new Resource().setName(name); +} + +export function crc32CheckSum(bytes: Buffer) { + return `${crc32(bytes)}`; +} + +export function md5CheckSum(bytes: Uint8Array) { + return createHash('md5').update(bytes).digest('hex') + .toUpperCase(); +} + +export function sha1CheckSum(bytes: Uint8Array) { + return createHash('sha1').update(bytes).digest('hex') + .toUpperCase(); +} + +// k0: 0x0706050403020100L, k1: 0x0f0e0d0c0b0a0908L +const SIP_HASH_24_KEY = Buffer.from([ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +]); +/** + * Java: Hashing.sipHash24().hashBytes(messageGroup.getBytes(StandardCharsets.UTF_8)).asLong() + */ +export function calculateStringSipHash24(value: string) { + const hash = siphash24(Buffer.from(value), SIP_HASH_24_KEY); + return Buffer.from(hash).readBigUInt64BE(); +} diff --git a/nodejs/test/consumer/SimpleConsumer.test.ts b/nodejs/test/consumer/SimpleConsumer.test.ts new file mode 100644 index 000000000..8e375a90d --- /dev/null +++ b/nodejs/test/consumer/SimpleConsumer.test.ts @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { randomUUID } from 'node:crypto'; +import { strict as assert } from 'node:assert'; +import { + SimpleConsumer, FilterExpression, + Producer, +} from '../../src'; +import { topics, endpoints, sessionCredentials } from '../helper'; + +describe('test/consumer/SimpleConsumer.test.ts', () => { + let producer: Producer | null = null; + let simpleConsumer: SimpleConsumer | null = null; + afterEach(async () => { + if (producer) { + await producer.shutdown(); + producer = null; + } + if (simpleConsumer) { + await simpleConsumer.shutdown(); + simpleConsumer = null; + } + }); + + describe('start with sessionCredentials', () => { + it('should work', async () => { + if (!sessionCredentials) return; + simpleConsumer = new SimpleConsumer({ + endpoints, + sessionCredentials, + consumerGroup: 'nodejs-unittest-group', + subscriptions: new Map().set(topics.delay, FilterExpression.SUB_ALL), + }); + await simpleConsumer.startup(); + }); + + it('should fail when accessKey invalid', async () => { + if (!sessionCredentials) return; + simpleConsumer = new SimpleConsumer({ + endpoints, + sessionCredentials: { + ...sessionCredentials, + accessKey: 'wrong', + }, + consumerGroup: 'nodejs-unittest-group', + subscriptions: new Map().set(topics.delay, FilterExpression.SUB_ALL), + }); + await assert.rejects(async () => { + await simpleConsumer!.startup(); + }, /Startup the rocketmq client failed, .+? error=ForbiddenException: .+? Username is not matched/); + }); + + it('should fail when accessSecret invalid', async () => { + if (!sessionCredentials) return; + simpleConsumer = new SimpleConsumer({ + endpoints, + sessionCredentials: { + ...sessionCredentials, + accessSecret: 'wrong', + }, + consumerGroup: 'nodejs-unittest-group', + subscriptions: new Map().set(topics.delay, FilterExpression.SUB_ALL), + }); + await assert.rejects(async () => { + await simpleConsumer!.startup(); + }, /Startup the rocketmq client failed, .+? error=ForbiddenException: .+? Check signature failed for accessKey/); + }); + }); + + describe('receive() and ack()', () => { + it('should receive success', async () => { + const topic = topics.normal; + const tag = `nodejs-unittest-tag-${randomUUID()}`; + producer = new Producer({ + endpoints, + sessionCredentials, + }); + await producer.startup(); + simpleConsumer = new SimpleConsumer({ + endpoints, + sessionCredentials, + consumerGroup: `nodejs-unittest-group-${randomUUID()}`, + subscriptions: new Map().set(topic, new FilterExpression(tag)), + }); + await simpleConsumer.startup(); + const receipt = await producer.send({ + topic, + tag, + body: Buffer.from(JSON.stringify({ hello: 'world' })), + }); + assert(receipt.messageId); + const messages = await simpleConsumer.receive(20, 10000); + assert.equal(messages.length, 1); + assert.equal(messages[0].messageId, receipt.messageId); + + const max = 102; + for (let i = 0; i < max; i++) { + const receipt = await producer.send({ + topic, + tag, + body: Buffer.from(JSON.stringify({ hello: 'world' })), + }); + assert(receipt.messageId); + } + + let count = 0; + while (count < max) { + const messages = await simpleConsumer.receive(20, 10000); + console.log('#%s: receive %d new messages', count, messages.length); + for (const message of messages) { + assert.equal(message.topic, topic); + // console.log('#%s: %o, %o', count, message, message.body.toString()); + const msg = JSON.parse(message.body.toString()); + assert.deepEqual(msg, { hello: 'world' }); + assert.equal(message.tag, tag); + count++; + await simpleConsumer.ack(message); + } + } + assert.equal(count, max); + }); + }); +}); diff --git a/nodejs/test/helper.ts b/nodejs/test/helper.ts new file mode 100644 index 000000000..4a3deec26 --- /dev/null +++ b/nodejs/test/helper.ts @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { SessionCredentials } from '../src/client'; + +export const endpoints = process.env.ROCKETMQ_NODEJS_CLIENT_ENDPOINTS ?? 'localhost:8081'; +export const topics = { + normal: 'TopicTestForNormal', + fifo: 'TopicTestForFifo', + delay: 'TopicTestForDelay', + transaction: 'TopicTestForTransaction', +}; + +export const consumerGroup = process.env.ROCKETMQ_NODEJS_CLIENT_GROUP ?? 'nodejs-unittest-group'; + +export let sessionCredentials: SessionCredentials | undefined; +if (process.env.ROCKETMQ_NODEJS_CLIENT_KEY && process.env.ROCKETMQ_NODEJS_CLIENT_SECRET) { + sessionCredentials = { + accessKey: process.env.ROCKETMQ_NODEJS_CLIENT_KEY, + accessSecret: process.env.ROCKETMQ_NODEJS_CLIENT_SECRET, + }; +} diff --git a/nodejs/test/index.test.ts b/nodejs/test/index.test.ts new file mode 100644 index 000000000..7ed765d4d --- /dev/null +++ b/nodejs/test/index.test.ts @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { strict as assert } from 'node:assert'; +import { SimpleConsumer, Producer } from '../src'; + +describe('test/index.test.ts', () => { + it('should export work', () => { + assert(SimpleConsumer); + assert.equal(SimpleConsumer.prototype.clientId, undefined); + assert.equal(typeof SimpleConsumer.prototype.startup, 'function'); + + assert(Producer); + assert.equal(Producer.prototype.clientId, undefined); + assert.equal(typeof Producer.prototype.startup, 'function'); + }); +}); diff --git a/nodejs/test/message/MessageId.test.ts b/nodejs/test/message/MessageId.test.ts new file mode 100644 index 000000000..8914d5e55 --- /dev/null +++ b/nodejs/test/message/MessageId.test.ts @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { strict as assert } from 'node:assert'; +import { MessageIdFactory } from '../../src'; + +describe('test/message/MessageId.test.ts', () => { + describe('MessageIdFactory()', () => { + it('should decode success', async () => { + const messageId = MessageIdFactory.decode('0156F7E71C361B21BC024CCDBE00000000'); + // console.log('messageId %o, toString %s', messageId, messageId); + assert.equal(messageId.version, 1); + assert.equal(messageId.macAddress, '56f7e71c361b'); + assert.equal(messageId.processId, 8636); + assert.equal(messageId.timestamp, 38587838); + assert.equal(messageId.sequence, 0); + // support lower case + const messageId2 = MessageIdFactory.decode('0156F7E71C361B21BC024CCDBE00000000'.toLowerCase()); + assert.equal(messageId2.version, 1); + assert.equal(messageId2.toString().toUpperCase(), messageId.toString()); + }); + + it('should create success', async () => { + const messageId = MessageIdFactory.create(); + // console.log('messageId %o, toString %s', messageId, messageId); + assert.equal(messageId.version, 1); + const decodeMessageId = MessageIdFactory.decode(messageId.toString()); + assert.equal(decodeMessageId.toString(), messageId.toString()); + assert.deepEqual(decodeMessageId, messageId); + const messageId2 = MessageIdFactory.create(); + assert.equal(messageId2.sequence, messageId.sequence + 1); + // console.log('messageId2 %o, toString %s', messageId2, messageId2); + }); + }); +}); diff --git a/nodejs/test/producer/Producer.test.ts b/nodejs/test/producer/Producer.test.ts new file mode 100644 index 000000000..005cd3b94 --- /dev/null +++ b/nodejs/test/producer/Producer.test.ts @@ -0,0 +1,283 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { strict as assert } from 'node:assert'; +import { randomUUID } from 'node:crypto'; +import { NotFoundException, Producer, SimpleConsumer } from '../../src'; +import { TransactionResolution } from '../../proto/apache/rocketmq/v2/definition_pb'; +import { topics, endpoints, sessionCredentials, consumerGroup } from '../helper'; + +describe('test/producer/Producer.test.ts', () => { + let producer: Producer | null = null; + let simpleConsumer: SimpleConsumer | null = null; + afterEach(async () => { + if (producer) { + await producer.shutdown(); + producer = null; + } + if (simpleConsumer) { + await simpleConsumer.shutdown(); + simpleConsumer = null; + } + }); + + describe('startup()', () => { + it('should startup success', async () => { + producer = new Producer({ + endpoints, + sessionCredentials, + maxAttempts: 2, + }); + await producer.startup(); + const sendReceipt = await producer.send({ + topic: topics.normal, + tag: 'nodejs-unittest', + keys: [ + `foo-key-${Date.now()}`, + `bar-key-${Date.now()}`, + ], + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄', + now: Date(), + })), + }); + // console.log('sendReceipt: %o', sendReceipt); + assert(sendReceipt.offset >= 0); + assert.equal(typeof sendReceipt.messageId, 'string'); + assert.equal(sendReceipt.messageId, sendReceipt.transactionId); + }); + + it('should startup fail when topic not exists', async () => { + await assert.rejects(async () => { + producer = new Producer({ + topic: 'TopicTest-not-exists', + endpoints, + sessionCredentials, + maxAttempts: 2, + }); + await producer.startup(); + }, (err: any) => { + assert.match(err.message, /Startup the rocketmq client failed, clientId=[^,]+, error=NotFoundException/); + assert.equal(err.cause instanceof NotFoundException, true); + assert.equal(err.cause.name, 'NotFoundException'); + assert.equal(err.cause.code, 40402); + assert.match(err.cause.message, /CODE: 17 {2}DESC: No topic route info in name server for the topic: TopicTest-not-exists/); + return true; + }); + }); + }); + + describe('send()', () => { + it('should send normal message', async () => { + const topic = topics.normal; + const tag = `nodejs-unittest-tag-${randomUUID()}`; + producer = new Producer({ + endpoints, + sessionCredentials, + maxAttempts: 2, + }); + await producer.startup(); + const receipt = await producer.send({ + topic, + tag, + keys: [ + `foo-key-${Date.now()}`, + `bar-key-${Date.now()}`, + ], + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄', + now: Date(), + })), + }); + assert(receipt.messageId); + + simpleConsumer = new SimpleConsumer({ + consumerGroup, + endpoints, + sessionCredentials, + subscriptions: new Map().set(topic, tag), + awaitDuration: 3000, + }); + await simpleConsumer.startup(); + const messages = await simpleConsumer.receive(1, 10000); + assert.equal(messages.length, 1); + assert.equal(messages[0].messageId, receipt.messageId); + await simpleConsumer.ack(messages[0]); + }); + + it('should send delay message', async () => { + const topic = topics.delay; + const tag = `nodejs-unittest-tag-${randomUUID()}`; + producer = new Producer({ + endpoints, + sessionCredentials, + maxAttempts: 2, + }); + await producer.startup(); + const startTime = Date.now(); + const receipt = await producer.send({ + topic, + tag, + delay: 1000, + keys: [ + `foo-key-${Date.now()}`, + `bar-key-${Date.now()}`, + ], + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄', + now: Date(), + })), + }); + assert(receipt.messageId); + + simpleConsumer = new SimpleConsumer({ + consumerGroup, + endpoints, + sessionCredentials, + subscriptions: new Map().set(topic, tag), + awaitDuration: 3000, + }); + await simpleConsumer.startup(); + const messages = await simpleConsumer.receive(1, 10000); + assert.equal(messages.length, 1); + const message = messages[0]; + assert.equal(message.messageId, receipt.messageId); + assert(message.transportDeliveryTimestamp); + assert(message.transportDeliveryTimestamp.getTime() - startTime >= 1000); + await simpleConsumer.ack(message); + }); + + it('should send fifo message', async () => { + const topic = topics.fifo; + const tag = `nodejs-unittest-tag-${randomUUID()}`; + producer = new Producer({ + endpoints, + sessionCredentials, + maxAttempts: 2, + }); + await producer.startup(); + simpleConsumer = new SimpleConsumer({ + consumerGroup, + endpoints, + sessionCredentials, + subscriptions: new Map().set(topic, tag), + awaitDuration: 3000, + }); + await simpleConsumer.startup(); + + // skip the first message + await producer.send({ + topic, + tag, + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄, first', + now: Date(), + })), + messageGroup: 'fifoMessageGroup', + }); + await simpleConsumer.receive(1, 10000); + + const receipt1 = await producer.send({ + topic, + tag, + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄, first', + now: Date(), + })), + messageGroup: 'fifoMessageGroup', + }); + assert(receipt1.messageId); + const receipt2 = await producer.send({ + topic, + tag, + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄, second', + now: Date(), + })), + messageGroup: 'fifoMessageGroup', + }); + assert(receipt2.messageId); + + let messages = await simpleConsumer.receive(1, 10000); + assert.equal(messages.length, 1); + let message = messages[0]; + assert.equal(JSON.parse(message.body.toString()).hello, 'rocketmq-client-nodejs world 😄, first'); + assert.equal(message.messageId, receipt1.messageId); + assert(message.messageGroup); + assert.equal(message.messageGroup, 'fifoMessageGroup'); + assert.equal(message.properties.get('__SHARDINGKEY'), 'fifoMessageGroup'); + await simpleConsumer.ack(message); + + messages = await simpleConsumer.receive(1, 10000); + assert.equal(messages.length, 1); + message = messages[0]; + assert.equal(JSON.parse(message.body.toString()).hello, 'rocketmq-client-nodejs world 😄, second'); + assert.equal(message.messageId, receipt2.messageId); + assert(message.messageGroup); + assert.equal(message.messageGroup, 'fifoMessageGroup'); + assert.equal(message.properties.get('__SHARDINGKEY'), 'fifoMessageGroup'); + await simpleConsumer.ack(message); + }); + + it('should send transaction message', async () => { + const topic = topics.transaction; + const tag = `nodejs-unittest-tag-${randomUUID()}`; + producer = new Producer({ + endpoints, + sessionCredentials, + maxAttempts: 2, + checker: { + async check(messageView) { + console.log(messageView); + return TransactionResolution.COMMIT; + }, + }, + }); + await producer.startup(); + const transaction = producer.beginTransaction(); + const receipt = await producer.send({ + topic, + tag, + keys: [ + `foo-key-${Date.now()}`, + `bar-key-${Date.now()}`, + ], + body: Buffer.from(JSON.stringify({ + hello: 'rocketmq-client-nodejs world 😄', + now: Date(), + })), + }, transaction); + await transaction.commit(); + + simpleConsumer = new SimpleConsumer({ + consumerGroup, + endpoints, + sessionCredentials, + subscriptions: new Map().set(topic, tag), + awaitDuration: 3000, + }); + await simpleConsumer.startup(); + const messages = await simpleConsumer.receive(2, 10000); + assert.equal(messages.length, 1); + const message = messages[0]; + assert.equal(message.messageId, receipt.messageId); + // console.log(message); + assert.equal(message.properties.get('__transactionId__'), receipt.transactionId); + await simpleConsumer.ack(message); + }); + }); +}); diff --git a/nodejs/test/start-rocketmq.sh b/nodejs/test/start-rocketmq.sh new file mode 100755 index 000000000..20d540deb --- /dev/null +++ b/nodejs/test/start-rocketmq.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Download rocketmq binary and start it +curl https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip -o rocketmq-all-5.1.3-bin-release.zip +unzip rocketmq-all-5.1.3-bin-release.zip +cd rocketmq-all-5.1.3-bin-release + +nohup sh bin/mqnamesrv & +sleep 10 +tail -n 5 ~/logs/rocketmqlogs/namesrv.log + +nohup sh bin/mqbroker -n localhost:9876 --enable-proxy & +sleep 10 +tail -n 5 ~/logs/rocketmqlogs/proxy.log + +# Create Topics +sh bin/mqadmin statsAll -n localhost:9876 +sh bin/mqadmin topicList -n localhost:9876 -c DefaultCluster +# Normal Message +sh bin/mqadmin updatetopic -n localhost:9876 -t TopicTestForNormal -c DefaultCluster +# FIFO Message +sh bin/mqadmin updatetopic -n localhost:9876 -t TopicTestForFifo -c DefaultCluster -a +message.type=FIFO +# Delay Message +sh bin/mqadmin updatetopic -n localhost:9876 -t TopicTestForDelay -c DefaultCluster -a +message.type=DELAY +# Transaction Message +sh bin/mqadmin updatetopic -n localhost:9876 -t TopicTestForTransaction -c DefaultCluster -a +message.type=TRANSACTION diff --git a/nodejs/test/util/index.test.ts b/nodejs/test/util/index.test.ts new file mode 100644 index 000000000..1ee50348a --- /dev/null +++ b/nodejs/test/util/index.test.ts @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { strict as assert } from 'node:assert'; +import { + getTimestamp, + calculateStringSipHash24, +} from '../../src/util'; + +describe('test/util/index.test.ts', () => { + describe('getTimestamp()', () => { + it('should work', () => { + const timestamp = getTimestamp(); + assert(timestamp.seconds); + assert(timestamp.nanos); + }); + }); + + describe('calculateStringSipHash24()', () => { + it('should work', () => { + assert.equal(calculateStringSipHash24('foo哈哈😄2222哈哈'), 11716758754047899126n); + assert.equal(calculateStringSipHash24('foo哈哈😄2222哈哈') % 3n, 2n); + }); + }); +}); diff --git a/nodejs/tsconfig.json b/nodejs/tsconfig.json new file mode 100644 index 000000000..e1c650577 --- /dev/null +++ b/nodejs/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@eggjs/tsconfig", + "compilerOptions": { + "target": "ES2022", + "rootDir": "./src", + "outDir": "./dist", + "moduleResolution": "Node" + }, + "include": [ "src" ], + "exclude": [ "node_modules" ] +} diff --git a/nodejs/tsconfig.prod.json b/nodejs/tsconfig.prod.json new file mode 100644 index 000000000..baa555d27 --- /dev/null +++ b/nodejs/tsconfig.prod.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": [ "node_modules", "test" ] +}