{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":284624837,"defaultBranch":"master","name":"tokenizer","ownerLogin":"sugarme","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2020-08-03T06:47:01.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/15897993?v=4","public":true,"private":false,"isOrgOwned":false},"refInfo":{"name":"","listCacheKey":"v0:1687680991.0","currentOid":""},"activityList":{"items":[{"before":"dc1def5005707ca8653090313a065814c77f5541","after":"2b7834ea27fe6a468cce9f083e850e847edf145d","ref":"refs/heads/master","pushedAt":"2024-06-14T11:52:32.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #42 from yarcat/master\n\npretrained/tokenizer: fix FromFile, add FromReader","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2351822748\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/42\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/42/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/42\">#42</a> from yarcat/master"}},{"before":"7599e43c1a109579c0b929e9b6efbd46663346a8","after":"dc1def5005707ca8653090313a065814c77f5541","ref":"refs/heads/master","pushedAt":"2024-06-14T11:50:26.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #41 from cwarden/fix-missing-pairEncoding\n\nFix TruncateEncodings for No Pair Encoding","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2349436733\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/41\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/41/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/41\">#41</a> from cwarden/fix-missing-pairEncoding"}},{"before":"448e79b1ed65947b8c6343bf9aa39e78364f45c8","after":"7599e43c1a109579c0b929e9b6efbd46663346a8","ref":"refs/heads/master","pushedAt":"2024-02-24T23:54:48.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #40 from jackielii/safer-input\n\nuse private() for better type safety","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2150918685\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/40\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/40/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/40\">#40</a> from jackielii/safer-input"}},{"before":"dcc38d367b545c788da8823adf4f1d675e48ec72","after":"448e79b1ed65947b8c6343bf9aa39e78364f45c8","ref":"refs/heads/master","pushedAt":"2023-08-29T21:49:35.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #33 from kasra73/master","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1872334745\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/33\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/33/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/33\">#33</a> from kasra73/master"}},{"before":"7a17b4e2319b9ea61fedaba99ca3f3cd6ea77686","after":"dcc38d367b545c788da8823adf4f1d675e48ec72","ref":"refs/heads/master","pushedAt":"2023-08-23T03:48:15.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge branch 'master' of ssh://github.com/sugarme/tokenizer","shortMessageHtmlLink":"Merge branch 'master' of ssh://github.com/sugarme/tokenizer"}},{"before":"590c8f1f6361852994cdfa2d4a28de284ad5b7b3","after":"7a17b4e2319b9ea61fedaba99ca3f3cd6ea77686","ref":"refs/heads/master","pushedAt":"2023-07-05T04:18:05.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #30 from yujong-lee/fix-pad","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1788654452\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/30\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/30/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/30\">#30</a> from yujong-lee/fix-pad"}},{"before":"1f25ffea0f6c5105f09dc80d4c9949344f0a516f","after":"590c8f1f6361852994cdfa2d4a28de284ad5b7b3","ref":"refs/heads/master","pushedAt":"2023-07-04T12:26:05.000Z","pushType":"pr_merge","commitsCount":4,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #29 from yujong-lee/sbert\n\nSupport sentence-transformers","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1787475100\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/29\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/29/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/29\">#29</a> from yujong-lee/sbert"}},{"before":"0e1bd3b838e43a08f40fd96c3cf684ec365bacff","after":"1f25ffea0f6c5105f09dc80d4c9949344f0a516f","ref":"refs/heads/master","pushedAt":"2023-06-25T08:15:55.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"bump new version","shortMessageHtmlLink":"bump new version"}},{"before":"584d1ecfe0af680a6cfe485b8b0d5d9245cc4d70","after":null,"ref":"refs/heads/decoder","pushedAt":"2023-06-25T08:14:47.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"}},{"before":"9029df54f6f7e79e829d644bd8a9368f6ff1bbfb","after":"0e1bd3b838e43a08f40fd96c3cf684ec365bacff","ref":"refs/heads/master","pushedAt":"2023-06-25T08:14:41.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #28 from sugarme/decoder\n\nfixed pretrained/createReplaceDecoder","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1773132225\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/28\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/28/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/28\">#28</a> from sugarme/decoder"}},{"before":null,"after":"584d1ecfe0af680a6cfe485b8b0d5d9245cc4d70","ref":"refs/heads/decoder","pushedAt":"2023-06-25T08:14:11.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"fixed pretrained/createReplaceDecoder","shortMessageHtmlLink":"fixed pretrained/createReplaceDecoder"}},{"before":"e78f6523e1ecdc78302bdd7a0cb7bbb39e38841a","after":"9029df54f6f7e79e829d644bd8a9368f6ff1bbfb","ref":"refs/heads/master","pushedAt":"2023-06-24T05:16:02.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"bump new version","shortMessageHtmlLink":"bump new version"}},{"before":"6225518c11f5a84be6c1159926e5866255545bfa","after":null,"ref":"refs/heads/pretrained","pushedAt":"2023-06-24T05:13:58.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"}},{"before":"0ab2e3ba834bbb7333b24b4e13231fec77d65502","after":"e78f6523e1ecdc78302bdd7a0cb7bbb39e38841a","ref":"refs/heads/master","pushedAt":"2023-06-24T05:13:44.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #27 from sugarme/pretrained\n\nfixed bugs at pretrained.FromFile","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1772462234\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/27\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/27/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/27\">#27</a> from sugarme/pretrained"}},{"before":null,"after":"6225518c11f5a84be6c1159926e5866255545bfa","ref":"refs/heads/pretrained","pushedAt":"2023-06-24T05:11:54.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"fixed bugs at pretrained.FromFile","shortMessageHtmlLink":"fixed bugs at pretrained.FromFile"}},{"before":"782839f48ef1d0bf138527b8d826989a8d1d72cf","after":"0ab2e3ba834bbb7333b24b4e13231fec77d65502","ref":"refs/heads/master","pushedAt":"2023-06-21T13:05:06.688Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"tag new version","shortMessageHtmlLink":"tag new version"}},{"before":"e205bac16d2c9da77c019b05aba80de456544b53","after":null,"ref":"refs/heads/decoder","pushedAt":"2023-06-21T13:02:42.845Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"}},{"before":"58656dfc9012e55c93f55bfca007f7e49b589628","after":"782839f48ef1d0bf138527b8d826989a8d1d72cf","ref":"refs/heads/master","pushedAt":"2023-06-21T13:02:31.647Z","pushType":"pr_merge","commitsCount":18,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"Merge pull request #25 from sugarme/decoder\n\nupdated pretrained","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1767555634\" data-permission-text=\"Title is private\" data-url=\"https://github.com/sugarme/tokenizer/issues/25\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/sugarme/tokenizer/pull/25/hovercard\" href=\"https://github.com/sugarme/tokenizer/pull/25\">#25</a> from sugarme/decoder"}},{"before":"8f58d0717613645baa7d1bb645b1f6c79fbfd561","after":"e205bac16d2c9da77c019b05aba80de456544b53","ref":"refs/heads/decoder","pushedAt":"2023-06-21T12:53:50.284Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"fixed normalizer.NFC()","shortMessageHtmlLink":"fixed normalizer.NFC()"}},{"before":"1585cf07718a997e473c33136a5e0af15fdc22c7","after":"8f58d0717613645baa7d1bb645b1f6c79fbfd561","ref":"refs/heads/decoder","pushedAt":"2023-06-21T12:19:29.584Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"added completed list of pretokenizers","shortMessageHtmlLink":"added completed list of pretokenizers"}},{"before":"deea443c0667ee7159783157e98d760623970ebb","after":"1585cf07718a997e473c33136a5e0af15fdc22c7","ref":"refs/heads/decoder","pushedAt":"2023-06-20T12:40:12.366Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: pretokenizer","shortMessageHtmlLink":"WIP: pretokenizer"}},{"before":"c3f59957f644001d226bf7636ffd585035724357","after":"deea443c0667ee7159783157e98d760623970ebb","ref":"refs/heads/decoder","pushedAt":"2023-06-19T13:13:56.092Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: pretrained/processor","shortMessageHtmlLink":"WIP: pretrained/processor"}},{"before":"efee0cb0a17d5b73fb7b35bb91c81a35a21bf33e","after":"c3f59957f644001d226bf7636ffd585035724357","ref":"refs/heads/decoder","pushedAt":"2023-06-14T08:34:53.256Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: TemplateProcessing","shortMessageHtmlLink":"WIP: TemplateProcessing"}},{"before":"269720b6760db84fe03632dfd93af067d7c4206b","after":"efee0cb0a17d5b73fb7b35bb91c81a35a21bf33e","ref":"refs/heads/decoder","pushedAt":"2023-06-10T13:59:39.770Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: processor.Sequence","shortMessageHtmlLink":"WIP: processor.Sequence"}},{"before":"6382a37d2a8c1ab48270ee9a81cb6f4ba12acd54","after":"269720b6760db84fe03632dfd93af067d7c4206b","ref":"refs/heads/decoder","pushedAt":"2023-06-09T12:55:42.014Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: Create Tokenizer FromFile continue...","shortMessageHtmlLink":"WIP: Create Tokenizer FromFile continue..."}},{"before":"5ee01cc52354ab4ef821932af4b2c0dbe18c04bf","after":"6382a37d2a8c1ab48270ee9a81cb6f4ba12acd54","ref":"refs/heads/decoder","pushedAt":"2023-06-09T09:49:29.787Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: added CreateNormalizer","shortMessageHtmlLink":"WIP: added CreateNormalizer"}},{"before":"cfab7d142a6c9dd4f897902fdf2fbb1b03339c1f","after":"5ee01cc52354ab4ef821932af4b2c0dbe18c04bf","ref":"refs/heads/decoder","pushedAt":"2023-06-09T05:30:38.217Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: pretrained.FromFile API","shortMessageHtmlLink":"WIP: pretrained.FromFile API"}},{"before":"3f282ceb566e104bd36500bfe96d1a2cc23f3fd2","after":"cfab7d142a6c9dd4f897902fdf2fbb1b03339c1f","ref":"refs/heads/decoder","pushedAt":"2023-06-09T01:37:36.275Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: refactor constructing Tokenizer from json file","shortMessageHtmlLink":"WIP: refactor constructing Tokenizer from json file"}},{"before":"375bb03642205edc5aece56a6cae174129460925","after":"3f282ceb566e104bd36500bfe96d1a2cc23f3fd2","ref":"refs/heads/decoder","pushedAt":"2023-06-08T13:24:28.003Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"WIP: New tokenizer from tokenizer.json","shortMessageHtmlLink":"WIP: New tokenizer from tokenizer.json"}},{"before":"bd92aadf929a2baed278778ba9ca930a4f913256","after":"375bb03642205edc5aece56a6cae174129460925","ref":"refs/heads/decoder","pushedAt":"2023-06-08T08:03:17.075Z","pushType":"push","commitsCount":1,"pusher":{"login":"sugarme","name":"sugarme","path":"/sugarme","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15897993?s=80&v=4"},"commit":{"message":"added google sentencepiece precompiled and other normalizers","shortMessageHtmlLink":"added google sentencepiece precompiled and other normalizers"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEZWlSmwA","startCursor":null,"endCursor":null}},"title":"Activity · sugarme/tokenizer"}