{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":564234132,"defaultBranch":"master","name":"scraper","ownerLogin":"bitcoinsearch","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2022-11-10T09:31:39.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/122760181?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1724606929.0","currentOid":""},"activityList":{"items":[{"before":"8980f03ece83fe49512d8089b7a27535764305f5","after":"622e60c6e93ee1d6832fa93be301d5980d7eac05","ref":"refs/heads/ai-scraper","pushedAt":"2024-08-29T10:04:12.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Tested answer extraction on StackExchange sites\n- added class name to prompt","shortMessageHtmlLink":"Tested answer extraction on StackExchange sites"}},{"before":"0367743b7dd48ef35e539b0c93659c256a2810ec","after":"8980f03ece83fe49512d8089b7a27535764305f5","ref":"refs/heads/ai-scraper","pushedAt":"2024-08-27T14:00:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Update Readme.md of AI scraper","shortMessageHtmlLink":"Update Readme.md of AI scraper"}},{"before":"2d3783b51ce10a79cebcf8f5dabf740d91c50dc5","after":"0367743b7dd48ef35e539b0c93659c256a2810ec","ref":"refs/heads/ai-scraper","pushedAt":"2024-08-25T18:09:01.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Implemented AI Scraper with ScrapeGraph\n\n- Developed initial pipeline with ScrapeGraph; optimized with custom graphs and script caching\n- Added rule-based HTML extraction and postprocessing\n- Integrated LLMs for component selection and tested various models\n- Fixed bugs and improved accuracy, including script generation and chunk-based processing\n- Updated HTML sanitization and data handling","shortMessageHtmlLink":"Implemented AI Scraper with ScrapeGraph"}},{"before":"6abf2f72411df96c6180ceec490a614c4841dc6b","after":null,"ref":"refs/heads/authors-synonyms","pushedAt":"2024-08-25T17:28:49.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"}},{"before":null,"after":"2d3783b51ce10a79cebcf8f5dabf740d91c50dc5","ref":"refs/heads/ai-scraper","pushedAt":"2024-08-25T17:26:15.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"resolve UnboundLocalError\n\n- updated variable name in `parse_markdown` function","shortMessageHtmlLink":"resolve UnboundLocalError"}},{"before":"b73a9996460d98598734c5cfac1e7dc64411d4ec","after":"ab6249e6e8a4124081df5d5ef1f500882b2a0d4e","ref":"refs/heads/scraper-logs","pushedAt":"2024-08-01T08:35:07.000Z","pushType":"push","commitsCount":12,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Save individual scraper logs in CSV file","shortMessageHtmlLink":"Save individual scraper logs in CSV file"}},{"before":"b206b67920cf83d41d704c68fdb6f0cd9b13619a","after":"2d3783b51ce10a79cebcf8f5dabf740d91c50dc5","ref":"refs/heads/master","pushedAt":"2024-08-01T08:09:02.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"resolve UnboundLocalError\n\n- updated variable name in `parse_markdown` function","shortMessageHtmlLink":"resolve UnboundLocalError"}},{"before":"a864b724ad8e01e3ead1453335b921265782610a","after":"b206b67920cf83d41d704c68fdb6f0cd9b13619a","ref":"refs/heads/master","pushedAt":"2024-07-29T09:22:11.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Merge pull request #78 from bitcoinsearch/authors-synonyms\n\n Handle Elasticsearch document version conflicts","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2434899029\" data-permission-text=\"Title is private\" data-url=\"https://github.com/bitcoinsearch/scraper/issues/78\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/bitcoinsearch/scraper/pull/78/hovercard\" href=\"https://github.com/bitcoinsearch/scraper/pull/78\">#78</a> from bitcoinsearch/authors-synonyms"}},{"before":"12a87092bcd386d190125ea05a5d0cb9d802a712","after":"6abf2f72411df96c6180ceec490a614c4841dc6b","ref":"refs/heads/authors-synonyms","pushedAt":"2024-07-29T09:19:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Added retry logic to handle Elasticsearch version conflicts in update_authors_names_from_es","shortMessageHtmlLink":"Added retry logic to handle Elasticsearch version conflicts in update…"}},{"before":"402bd6ad2ba0461a445007cb96c879c0e86bbc45","after":"a864b724ad8e01e3ead1453335b921265782610a","ref":"refs/heads/master","pushedAt":"2024-07-29T09:09:39.000Z","pushType":"pr_merge","commitsCount":4,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Merge pull request #77 from bitcoinsearch/authors-synonyms\n\nAuthor Synonyms","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2425395660\" data-permission-text=\"Title is private\" data-url=\"https://github.com/bitcoinsearch/scraper/issues/77\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/bitcoinsearch/scraper/pull/77/hovercard\" href=\"https://github.com/bitcoinsearch/scraper/pull/77\">#77</a> from bitcoinsearch/authors-synonyms"}},{"before":"96d4d713c752251a06b69c3e7eb2a6fd31f83d9b","after":"12a87092bcd386d190125ea05a5d0cb9d802a712","ref":"refs/heads/authors-synonyms","pushedAt":"2024-07-29T09:08:25.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"removed unused functions","shortMessageHtmlLink":"removed unused functions"}},{"before":"1de30535b88e95b2eae8dece0c5cc85c1d9ddf61","after":"96d4d713c752251a06b69c3e7eb2a6fd31f83d9b","ref":"refs/heads/authors-synonyms","pushedAt":"2024-07-26T12:38:01.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Author Synonyms:\n- find-and-replace author names in ES index based on author-synonyms.csv","shortMessageHtmlLink":"Author Synonyms:"}},{"before":"402bd6ad2ba0461a445007cb96c879c0e86bbc45","after":"1de30535b88e95b2eae8dece0c5cc85c1d9ddf61","ref":"refs/heads/authors-synonyms","pushedAt":"2024-07-23T14:39:01.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Author Synonyms:\n- update author synonyms with default author name\n- daily GitHub Actions file","shortMessageHtmlLink":"Author Synonyms:"}},{"before":null,"after":"402bd6ad2ba0461a445007cb96c879c0e86bbc45","ref":"refs/heads/authors-synonyms","pushedAt":"2024-07-23T14:13:58.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Merge pull request #75 from kouloumos/parse-markdown\n\nRefactor `parse_markdown` and extract to utils","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2415813547\" data-permission-text=\"Title is private\" data-url=\"https://github.com/bitcoinsearch/scraper/issues/75\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/bitcoinsearch/scraper/pull/75/hovercard\" href=\"https://github.com/bitcoinsearch/scraper/pull/75\">#75</a> from kouloumos/parse-markdown"}},{"before":"81de5485a859fc7f8b63929ed6b31c09aa93274d","after":"402bd6ad2ba0461a445007cb96c879c0e86bbc45","ref":"refs/heads/master","pushedAt":"2024-07-19T11:16:13.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"kouloumos","name":"Andreas","path":"/kouloumos","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/18506343?s=80&v=4"},"commit":{"message":"Merge pull request #75 from kouloumos/parse-markdown\n\nRefactor `parse_markdown` and extract to utils","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2415813547\" data-permission-text=\"Title is private\" data-url=\"https://github.com/bitcoinsearch/scraper/issues/75\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/bitcoinsearch/scraper/pull/75/hovercard\" href=\"https://github.com/bitcoinsearch/scraper/pull/75\">#75</a> from kouloumos/parse-markdown"}},{"before":"8abd8eda9812e74f970cf83cd8debfccd8845381","after":"4741a7f0f25b8838221cb88a2fbec283db3f3013","ref":"refs/heads/bitcointalk-author-synonyms","pushedAt":"2024-07-16T17:34:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"BitcoinTalk:\n- integrate author synonyms in scraper","shortMessageHtmlLink":"BitcoinTalk:"}},{"before":"81de5485a859fc7f8b63929ed6b31c09aa93274d","after":"8abd8eda9812e74f970cf83cd8debfccd8845381","ref":"refs/heads/bitcointalk-author-synonyms","pushedAt":"2024-07-16T14:33:16.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"BitcoinTalk:\n- update author names as per author-synonyms","shortMessageHtmlLink":"BitcoinTalk:"}},{"before":null,"after":"81de5485a859fc7f8b63929ed6b31c09aa93274d","ref":"refs/heads/bitcointalk-author-synonyms","pushedAt":"2024-07-16T13:09:01.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Merge pull request #72 from bitcoinsearch/70-include-source-metadata\n\nbitcointranscripts: Include Source Metadata","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2353798306\" data-permission-text=\"Title is private\" data-url=\"https://github.com/bitcoinsearch/scraper/issues/72\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/bitcoinsearch/scraper/pull/72/hovercard\" href=\"https://github.com/bitcoinsearch/scraper/pull/72\">#72</a> from bitcoinsearch/70-include-source-metadata"}},{"before":"cd93099f5fc1b500a1505f9c5c43567776cf0175","after":null,"ref":"refs/heads/feature/69-add-ai-transcript-flag","pushedAt":"2024-07-16T13:07:49.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"}},{"before":"350c0779e41935e9a8ddb185759c7aa88d62451b","after":null,"ref":"refs/heads/70-include-source-metadata","pushedAt":"2024-07-16T13:07:40.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"}},{"before":"cdabfd3e53e3bbc20d55727987bf830815b383b4","after":"70bed34072fa845b7a0eaeff9a53dd6a0f38d775","ref":"refs/heads/pdf-scraper","pushedAt":"2024-07-11T10:13:39.000Z","pushType":"push","commitsCount":22,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Merge branch 'master' of urvish_gt:bitcoinsearch/scraper into pdf-scraper","shortMessageHtmlLink":"Merge branch 'master' of urvish_gt:bitcoinsearch/scraper into pdf-scr…"}},{"before":"81de5485a859fc7f8b63929ed6b31c09aa93274d","after":"b73a9996460d98598734c5cfac1e7dc64411d4ec","ref":"refs/heads/scraper-logs","pushedAt":"2024-07-05T10:50:11.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Save scraper logs in csv","shortMessageHtmlLink":"Save scraper logs in csv"}},{"before":"0e9c2d8f0eb31915b191cfeabe9a3c2674d06589","after":"81de5485a859fc7f8b63929ed6b31c09aa93274d","ref":"refs/heads/scraper-logs","pushedAt":"2024-07-02T10:03:50.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Merge pull request #72 from bitcoinsearch/70-include-source-metadata\n\nbitcointranscripts: Include Source Metadata","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2353798306\" data-permission-text=\"Title is private\" data-url=\"https://github.com/bitcoinsearch/scraper/issues/72\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/bitcoinsearch/scraper/pull/72/hovercard\" href=\"https://github.com/bitcoinsearch/scraper/pull/72\">#72</a> from bitcoinsearch/70-include-source-metadata"}},{"before":"0e9c2d8f0eb31915b191cfeabe9a3c2674d06589","after":"81de5485a859fc7f8b63929ed6b31c09aa93274d","ref":"refs/heads/master","pushedAt":"2024-07-02T09:57:29.000Z","pushType":"pr_merge","commitsCount":3,"pusher":{"login":"kouloumos","name":"Andreas","path":"/kouloumos","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/18506343?s=80&v=4"},"commit":{"message":"Merge pull request #72 from bitcoinsearch/70-include-source-metadata\n\nbitcointranscripts: Include Source Metadata","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2353798306\" data-permission-text=\"Title is private\" data-url=\"https://github.com/bitcoinsearch/scraper/issues/72\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/bitcoinsearch/scraper/pull/72/hovercard\" href=\"https://github.com/bitcoinsearch/scraper/pull/72\">#72</a> from bitcoinsearch/70-include-source-metadata"}},{"before":null,"after":"0e9c2d8f0eb31915b191cfeabe9a3c2674d06589","ref":"refs/heads/scraper-logs","pushedAt":"2024-07-02T07:51:03.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"old mailing list: update 'url' and 'thread_url' logic","shortMessageHtmlLink":"old mailing list: update 'url' and 'thread_url' logic"}},{"before":"2f64160a94d0615aea71ee1ad95b26f5e9a36c39","after":"0e9c2d8f0eb31915b191cfeabe9a3c2674d06589","ref":"refs/heads/master","pushedAt":"2024-06-25T18:53:07.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"old mailing list: update 'url' and 'thread_url' logic","shortMessageHtmlLink":"old mailing list: update 'url' and 'thread_url' logic"}},{"before":"c437d2668235501a4490bbdd0116891026ce9d41","after":"2f64160a94d0615aea71ee1ad95b26f5e9a36c39","ref":"refs/heads/master","pushedAt":"2024-06-24T15:14:46.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"old mailing list: update 'url' and 'thread_url' logic","shortMessageHtmlLink":"old mailing list: update 'url' and 'thread_url' logic"}},{"before":"f49b2a1dc6353228e8929d4c597ec709af19eefe","after":"c437d2668235501a4490bbdd0116891026ce9d41","ref":"refs/heads/master","pushedAt":"2024-06-23T15:22:45.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"old mailing list: update 'url' and 'thread_url' logic","shortMessageHtmlLink":"old mailing list: update 'url' and 'thread_url' logic"}},{"before":"83241f3d2e9ec4c13e5e57ea6b5ffb91a4e22e82","after":"f49b2a1dc6353228e8929d4c597ec709af19eefe","ref":"refs/heads/master","pushedAt":"2024-06-19T18:25:15.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"Update bitcointalk.yml\n\nfix python-version in workflow file","shortMessageHtmlLink":"Update bitcointalk.yml"}},{"before":"f5b9ab7bf90b6adf5493b573c1e8226b81f99708","after":"350c0779e41935e9a8ddb185759c7aa88d62451b","ref":"refs/heads/70-include-source-metadata","pushedAt":"2024-06-17T17:56:16.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"urvishp80","name":"Urviskumar","path":"/urvishp80","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/26462893?s=80&v=4"},"commit":{"message":"bitcointranscripts: Boolean for \"needs_review\" field","shortMessageHtmlLink":"bitcointranscripts: Boolean for \"needs_review\" field"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOC0yOVQxMDowNDoxMi4wMDAwMDBazwAAAASnc4O9","startCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOC0yOVQxMDowNDoxMi4wMDAwMDBazwAAAASnc4O9","endCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wNi0xN1QxNzo1NjoxNi4wMDAwMDBazwAAAARnp1l0"}},"title":"Activity · bitcoinsearch/scraper"}