{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":473601588,"defaultBranch":"main","name":"lm-evaluation-harness","ownerLogin":"OpenGPTX","currentUserCanPush":false,"isFork":true,"isEmpty":false,"createdAt":"2022-03-24T12:43:03.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/98085413?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1725896504.0","currentOid":""},"activityList":{"items":[{"before":null,"after":"ce167c32e9df1c623d2bb4202cf08eba67179261","ref":"refs/heads/crowspairs-es","pushedAt":"2024-09-09T15:41:44.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"NAM00","name":"Nazia Afsan Mowmita","path":"/NAM00","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/66006043?s=80&v=4"},"commit":{"message":"Spanish crowspairs Eval","shortMessageHtmlLink":"Spanish crowspairs Eval"}},{"before":"43468b998f0cc1db09d8dd3a252470cc63728eec","after":"0ac93dcd9ec96bd1214fb6e966a2936fbe6fbb08","ref":"refs/heads/polyglotoxicityprompts","pushedAt":"2024-08-23T15:43:42.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"linting","shortMessageHtmlLink":"linting"}},{"before":null,"after":"43468b998f0cc1db09d8dd3a252470cc63728eec","ref":"refs/heads/polyglotoxicityprompts","pushedAt":"2024-08-23T15:43:23.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"Merge branch 'upgrade_v0.4.3'","shortMessageHtmlLink":"Merge branch 'upgrade_v0.4.3'"}},{"before":"0ac93dcd9ec96bd1214fb6e966a2936fbe6fbb08","after":null,"ref":"refs/heads/polyglotoxicityprompts","pushedAt":"2024-08-23T15:40:17.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"}},{"before":"43468b998f0cc1db09d8dd3a252470cc63728eec","after":"0ac93dcd9ec96bd1214fb6e966a2936fbe6fbb08","ref":"refs/heads/polyglotoxicityprompts","pushedAt":"2024-08-23T15:36:45.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"linting","shortMessageHtmlLink":"linting"}},{"before":null,"after":"43468b998f0cc1db09d8dd3a252470cc63728eec","ref":"refs/heads/polyglotoxicityprompts","pushedAt":"2024-08-23T15:21:56.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"Merge branch 'upgrade_v0.4.3'","shortMessageHtmlLink":"Merge branch 'upgrade_v0.4.3'"}},{"before":null,"after":"5b6f023c865c23e3986a359e475948f887d25951","ref":"refs/heads/upgrade_main_42dc24","pushedAt":"2024-08-02T08:03:52.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Merge remote-tracking branch 'upstream/main' into upgrade_main_42dc24","shortMessageHtmlLink":"Merge remote-tracking branch 'upstream/main' into upgrade_main_42dc24"}},{"before":"467e766136436f8fc4f29960f0c301fcceaf114d","after":"43468b998f0cc1db09d8dd3a252470cc63728eec","ref":"refs/heads/main","pushedAt":"2024-07-25T11:23:45.000Z","pushType":"push","commitsCount":211,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Merge branch 'upgrade_v0.4.3'","shortMessageHtmlLink":"Merge branch 'upgrade_v0.4.3'"}},{"before":null,"after":"ebb5f2d188fc000a8bafaa08119c6f8ae12f08a2","ref":"refs/heads/upgrade_v0.4.3","pushedAt":"2024-07-22T18:14:47.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"HF model: Added 'nccl_timeout' model arg","shortMessageHtmlLink":"HF model: Added 'nccl_timeout' model arg"}},{"before":"6092ac50514b90d43385c65012164956f42d4747","after":"6cc336044bde4621a488dcfaf272c0bdd5b6df4e","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-07-22T09:47:24.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"HF model: Added 'nccl_timeout' model arg","shortMessageHtmlLink":"HF model: Added 'nccl_timeout' model arg"}},{"before":null,"after":"467e766136436f8fc4f29960f0c301fcceaf114d","ref":"refs/heads/main","pushedAt":"2024-07-16T13:46:35.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Merge branch 'yaml_multilingual_tasks'","shortMessageHtmlLink":"Merge branch 'yaml_multilingual_tasks'"}},{"before":null,"after":"f6b336fe999e01ccc354de4f4ce5bfc0731112cf","ref":"refs/heads/master_bak","pushedAt":"2024-07-16T13:31:30.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Merge pull request #102 from OpenGPTX/malteos-patch-1\n\nDisable unit tests","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2005015156\" data-permission-text=\"Title is private\" data-url=\"https://github.com/OpenGPTX/lm-evaluation-harness/issues/102\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/OpenGPTX/lm-evaluation-harness/pull/102/hovercard\" href=\"https://github.com/OpenGPTX/lm-evaluation-harness/pull/102\">#102</a> from OpenGPTX/malteos-patch-1"}},{"before":"485acdda7dd7aa350902097872f1930e69868863","after":"6092ac50514b90d43385c65012164956f42d4747","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-06-28T13:00:23.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"added lvs config for belebele, also renamed config files","shortMessageHtmlLink":"added lvs config for belebele, also renamed config files"}},{"before":"7447360d18b6d785dbfda2889bb38d36cc163045","after":"485acdda7dd7aa350902097872f1930e69868863","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-06-28T12:54:07.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"added latvian flores200 configs","shortMessageHtmlLink":"added latvian flores200 configs"}},{"before":null,"after":"c96afbe25da0c3e39ce298a6be4a790eeb4835f8","ref":"refs/heads/mt_bench_x","pushedAt":"2024-05-22T08:09:41.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"lllAlexanderlll","name":"Alexander Weber","path":"/lllAlexanderlll","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12560547?s=80&v=4"},"commit":{"message":"on hold: multi-turn evaluations not supported by lm-evaluation-harness by now.","shortMessageHtmlLink":"on hold: multi-turn evaluations not supported by lm-evaluation-harnes…"}},{"before":"f8e00f76771891444d6b418bd5c08ab4e0b6b290","after":"7447360d18b6d785dbfda2889bb38d36cc163045","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-04-17T09:02:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Added missing nll metric implementation","shortMessageHtmlLink":"Added missing nll metric implementation"}},{"before":"28bf93f96f073c1e7bfb8943ff7a7b3f5b0133e9","after":"f8e00f76771891444d6b418bd5c08ab4e0b6b290","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-04-15T19:01:54.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Added flores nll fix","shortMessageHtmlLink":"Added flores nll fix"}},{"before":"4fe393c71d7e5ed213ec53d97627b9477b3ed1f3","after":"28bf93f96f073c1e7bfb8943ff7a7b3f5b0133e9","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-04-15T18:31:42.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Preliminary implementation of flores and belebele","shortMessageHtmlLink":"Preliminary implementation of flores and belebele"}},{"before":"de88d6be02fef9d7292b8ca7956b45d5e02fb0c9","after":"4fe393c71d7e5ed213ec53d97627b9477b3ed1f3","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-22T14:14:12.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Tasks corrections: MMLU and GSM8k","shortMessageHtmlLink":"Tasks corrections: MMLU and GSM8k"}},{"before":"521504eb1b0dd2dc48287938d0796bb44e4f05a0","after":"de88d6be02fef9d7292b8ca7956b45d5e02fb0c9","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-18T12:05:19.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"Merge branch 'yaml_multilingual_tasks' of https://github.com/OpenGPTX/lm-evaluation-harness into yaml_multilingual_tasks","shortMessageHtmlLink":"Merge branch 'yaml_multilingual_tasks' of <a href=\"https://github.com/OpenGPTX/lm-evaluation-harness\">https://github.com/OpenGPTX…</a>"}},{"before":"e8d9f3853d6a19df192b19a09ab56e0e26fe4259","after":"521504eb1b0dd2dc48287938d0796bb44e4f05a0","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-18T11:09:41.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Task corrections: MMLU and GSM8k","shortMessageHtmlLink":"Task corrections: MMLU and GSM8k"}},{"before":"2fcfe02a3d95764396ef50c7ad5fd38799904afb","after":"e8d9f3853d6a19df192b19a09ab56e0e26fe4259","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-15T14:43:13.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"Merge branch 'yaml_multilingual_tasks' of https://github.com/OpenGPTX/lm-evaluation-harness into yaml_multilingual_tasks","shortMessageHtmlLink":"Merge branch 'yaml_multilingual_tasks' of <a href=\"https://github.com/OpenGPTX/lm-evaluation-harness\">https://github.com/OpenGPTX…</a>"}},{"before":"f96e5850825c8d71c32b4793070726f83493312e","after":"2fcfe02a3d95764396ef50c7ad5fd38799904afb","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-13T18:47:48.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Task fixes","shortMessageHtmlLink":"Task fixes"}},{"before":"93f549c61618efb72c5563a4ad8d7406ff30033c","after":"f96e5850825c8d71c32b4793070726f83493312e","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-13T18:27:58.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jjbuschhoff","name":null,"path":"/jjbuschhoff","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/138657490?s=80&v=4"},"commit":{"message":"added truthfulqax yaml configs","shortMessageHtmlLink":"added truthfulqax yaml configs"}},{"before":"25f0b2534c0400f2aabfca9ca7d874d12f425593","after":"93f549c61618efb72c5563a4ad8d7406ff30033c","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-13T15:39:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Task fixes","shortMessageHtmlLink":"Task fixes"}},{"before":"8fa981d50db4af7a980a0fd289594f8fe7adfabd","after":"25f0b2534c0400f2aabfca9ca7d874d12f425593","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-13T14:47:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Added ogx arcx tasks","shortMessageHtmlLink":"Added ogx arcx tasks"}},{"before":"cf9e190ecdfa30dd9b6496584cb799d6715910b6","after":"8fa981d50db4af7a980a0fd289594f8fe7adfabd","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-13T13:53:51.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Added ogx hellaswagx task","shortMessageHtmlLink":"Added ogx hellaswagx task"}},{"before":null,"after":"cf9e190ecdfa30dd9b6496584cb799d6715910b6","ref":"refs/heads/yaml_multilingual_tasks","pushedAt":"2024-03-13T11:59:56.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"KlaudiaTH","name":"Klaudia Thellmann","path":"/KlaudiaTH","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5738212?s=80&v=4"},"commit":{"message":"Added multilingual arc, hellaswag, mmlu and truthfulqa from EleutherAI upstream","shortMessageHtmlLink":"Added multilingual arc, hellaswag, mmlu and truthfulqa from EleutherA…"}},{"before":null,"after":"51c40fb87c3db679fd89de79c63343c914fb6c78","ref":"refs/heads/crowspairsit","pushedAt":"2024-03-07T11:40:10.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"NAM00","name":"Nazia Afsan Mowmita","path":"/NAM00","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/66006043?s=80&v=4"},"commit":{"message":"Italian task crowspairs","shortMessageHtmlLink":"Italian task crowspairs"}},{"before":null,"after":"e5cdfcfcf7ad956cd27c44fe461c6f816545f0b4","ref":"refs/heads/crowspairses","pushedAt":"2024-03-07T11:37:15.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"NAM00","name":"Nazia Afsan Mowmita","path":"/NAM00","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/66006043?s=80&v=4"},"commit":{"message":"Spanish task crowspairs","shortMessageHtmlLink":"Spanish task crowspairs"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0wOVQxNTo0MTo0NC4wMDAwMDBazwAAAASxFZaY","startCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0wOVQxNTo0MTo0NC4wMDAwMDBazwAAAASxFZaY","endCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wMy0wN1QxMTozNzoxNS4wMDAwMDBazwAAAAQO_K00"}},"title":"Activity · OpenGPTX/lm-evaluation-harness"}