diff --git a/README.md b/README.md index d5b08ff..0248da5 100644 --- a/README.md +++ b/README.md @@ -126,36 +126,36 @@ additional tasks have the following optional fields: Method | Download .jsonl file | Viewer link --------- | --------------------------------------------------------------------------------------------------------------------------------- | ----------- -BM25 | [trex_retrievals_bm25.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_bm25.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_bm25.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_bm25.jsonl%29) -Gecko | [trex_retrievals_gecko.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_gecko.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_gecko.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_gecko.jsonl%29) -TRAK | [trex_retrievals_trak.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_trak.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_trak.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_trak.jsonl%29) -Exp 1 | [trex_retrievals_exp1.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp1.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_exp1.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp1.jsonl%29) -Exp 2 | [trex_retrievals_exp2.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp2.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_exp2.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp2.jsonl%29) -Exp 3 | [trex_retrievals_exp3.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp3.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_exp3.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp3.jsonl%29) -Exp 4 | [trex_retrievals_exp4.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp4.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_exp4.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp4.jsonl%29) -Exp 5 | [trex_retrievals_exp5.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp5.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_exp5.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp5.jsonl%29) -TrackStar | [trex_retrievals_trackstar.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_trackstar.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Btrex_retrievals_trackstar.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_trackstar.jsonl%29) +BM25 | [trex_retrievals_bm25.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_bm25.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_bm25.jsonl) +Gecko | [trex_retrievals_gecko.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_gecko.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_gecko.jsonl) +TRAK | [trex_retrievals_trak.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_trak.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_trak.jsonl) +Exp 1 | [trex_retrievals_exp1.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp1.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp1.jsonl) +Exp 2 | [trex_retrievals_exp2.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp2.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp2.jsonl) +Exp 3 | [trex_retrievals_exp3.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp3.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp3.jsonl) +Exp 4 | [trex_retrievals_exp4.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp4.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp4.jsonl) +Exp 5 | [trex_retrievals_exp5.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_exp5.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_exp5.jsonl) +TrackStar | [trex_retrievals_trackstar.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/trex_retrievals_trackstar.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Ftrex_retrievals_trackstar.jsonl) **Table 2: T-REx facts, retrievals from C4** Method | Download .jsonl file | Viewer link -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ----------- -BM25 | [c4_trex_retrievals_bm25.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_bm25.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_trex_retrievals_bm25.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_bm25.jsonl%29) -Gecko | [c4_trex_retrievals_gecko.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_gecko.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_trex_retrievals_gecko.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_gecko.jsonl%29) -Gradient dot product | [c4_trex_retrievals_grad_dot.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_grad_dot.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_trex_retrievals_grad_dot.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_grad_dot.jsonl%29) -Gradient cosine | [c4_trex_retrievals_grad_cosine.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_grad_cosine.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_trex_retrievals_grad_cosine.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_grad_cosine.jsonl%29) -TrackStar | [c4_trex_retrievals_trackstar.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_trackstar.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_trex_retrievals_trackstar.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_trackstar.jsonl%29) +BM25 | [c4_trex_retrievals_bm25.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_bm25.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_bm25.jsonl) +Gecko | [c4_trex_retrievals_gecko.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_gecko.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_gecko.jsonl) +Gradient dot product | [c4_trex_retrievals_grad_dot.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_grad_dot.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_grad_dot.jsonl) +Gradient cosine | [c4_trex_retrievals_grad_cosine.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_grad_cosine.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_grad_cosine.jsonl) +TrackStar | [c4_trex_retrievals_trackstar.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_retrievals_trackstar.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_retrievals_trackstar.jsonl) **Appendix A.5: Additional tasks, retrievals from C4** Task | Download .jsonl file | Viewer link --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- -T-REx incorrect predictions | [c4_trex_incorrectpred_retrievals_trackstar.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_incorrectpred_retrievals_trackstar.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_trex_incorrectpred_retrievals_trackstar.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_incorrectpred_retrievals_trackstar.jsonl%29) -COPA | [c4_copa_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_copa_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_copa_retrievals_trackstar_nontaskspecific.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_copa_retrievals_trackstar_nontaskspecific.jsonl%29) -PIQA | [c4_piqa_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_piqa_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_piqa_retrievals_trackstar_nontaskspecific.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_piqa_retrievals_trackstar_nontaskspecific.jsonl%29) -Arithmetic word problems | [c4_arithmeticwordproblem_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_arithmeticwordproblem_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_arithmeticwordproblem_retrievals_trackstar_nontaskspecific.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_arithmeticwordproblem_retrievals_trackstar_nontaskspecific.jsonl%29) -Simple arithmetic | [c4_arithmetic_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_arithmetic_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_arithmetic_retrievals_trackstar_nontaskspecific.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_arithmetic_retrievals_trackstar_nontaskspecific.jsonl%29) -Story generation | [c4_storygeneration_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_storygeneration_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=%5Bc4_storygeneration_retrievals_trackstar_nontaskspecific.jsonl%5D%28https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_storygeneration_retrievals_trackstar_nontaskspecific.jsonl%29) +T-REx incorrect predictions | [c4_trex_incorrectpred_retrievals_trackstar.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_trex_incorrectpred_retrievals_trackstar.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_trex_incorrectpred_retrievals_trackstar.jsonl) +COPA | [c4_copa_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_copa_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_copa_retrievals_trackstar_nontaskspecific.jsonl) +PIQA | [c4_piqa_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_piqa_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_piqa_retrievals_trackstar_nontaskspecific.jsonl) +Arithmetic word problems | [c4_arithmeticwordproblem_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_arithmeticwordproblem_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_arithmeticwordproblem_retrievals_trackstar_nontaskspecific.jsonl) +Simple arithmetic | [c4_arithmetic_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_arithmetic_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_arithmetic_retrievals_trackstar_nontaskspecific.jsonl) +Story generation | [c4_storygeneration_retrievals_trackstar_nontaskspecific.jsonl](https://storage.googleapis.com/tda-resources/2410.17413/public/c4_storygeneration_retrievals_trackstar_nontaskspecific.jsonl) | [view in app](https://pair-code.github.io/pretraining-tda/demo/?jsonl_path=https%3A%2F%2Fstorage.googleapis.com%2Ftda-resources%2F2410.17413%2Fpublic%2Fc4_storygeneration_retrievals_trackstar_nontaskspecific.jsonl) ### T-REx sentences