From c9a8e1928444052ef267225071f62557e725765b Mon Sep 17 00:00:00 2001 From: Alex Mallen Date: Thu, 9 Nov 2023 19:40:37 +0000 Subject: [PATCH 1/4] fix sweep default --- elk/training/sweep.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index a4e5c97a..1902e0d7 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -1,4 +1,4 @@ -from dataclasses import InitVar, dataclass, replace +from dataclasses import InitVar, dataclass, replace, field import numpy as np import torch @@ -53,10 +53,11 @@ class Sweep: name: str | None = None # A bit of a hack to add all the command line arguments from Elicit - run_template: Elicit = Elicit( - data=Extract( - model="", - datasets=("",), + run_template: Elicit = field(default_factory=lambda: Elicit( + data=Extract( + model="", + datasets=("",), + ) ) ) From 611c93ca02294bd9593104c2db78bffe266f5204 Mon Sep 17 00:00:00 2001 From: Alex Mallen Date: Fri, 10 Nov 2023 17:29:59 +0000 Subject: [PATCH 2/4] Merge with EleutherELK --- elk/training/sweep.py | 11 ++--------- elk/training/train.py | 10 ++++++++++ pyproject.toml | 2 ++ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/elk/training/sweep.py b/elk/training/sweep.py index 1902e0d7..c9ca7fc9 100755 --- a/elk/training/sweep.py +++ b/elk/training/sweep.py @@ -1,4 +1,4 @@ -from dataclasses import InitVar, dataclass, replace, field +from dataclasses import InitVar, dataclass, field, replace import numpy as np import torch @@ -6,7 +6,6 @@ from transformers import AutoConfig from ..evaluation import Eval -from ..extraction import Extract from ..files import memorably_named_dir, sweeps_dir from ..plotting.visualize import visualize_sweep from ..training.eigen_reporter import EigenFitterConfig @@ -53,13 +52,7 @@ class Sweep: name: str | None = None # A bit of a hack to add all the command line arguments from Elicit - run_template: Elicit = field(default_factory=lambda: Elicit( - data=Extract( - model="", - datasets=("",), - ) - ) - ) + run_template: Elicit = field(default_factory=Elicit.default) def __post_init__(self, add_pooled: bool): if not self.datasets: diff --git a/elk/training/train.py b/elk/training/train.py index fb882240..3d00b54c 100644 --- a/elk/training/train.py +++ b/elk/training/train.py @@ -11,6 +11,7 @@ from simple_parsing import subgroups from simple_parsing.helpers.serialization import save +from ..extraction import Extract from ..metrics import evaluate_preds, to_one_hot from ..run import Run from ..training.supervised import train_supervised @@ -34,6 +35,15 @@ class Elicit(Run): cross-validation. Defaults to "single", which means to train a single classifier on the training data. "cv" means to use cross-validation.""" + @staticmethod + def default(): + return Elicit( + data=Extract( + model="", + datasets=("",), + ) + ) + def create_models_dir(self, out_dir: Path): lr_dir = None lr_dir = out_dir / "lr_models" diff --git a/pyproject.toml b/pyproject.toml index f3f16504..cb004269 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,8 @@ license = { text = "MIT License" } dependencies = [ # Allows us to use device_map in from_pretrained. Also needed for 8bit "accelerate", + # Already a dependency of datasets, but newer versions introduce breaking changes + "pyarrow==12.0.0", # For pseudolabel and prompt normalization. We're picky about the version because # the package isn't guaranteed to be stable yet. "concept-erasure==0.1.0", From 71d79a279045d9405a5333545dccdfe207ad6fbc Mon Sep 17 00:00:00 2001 From: Alex Mallen Date: Fri, 10 Nov 2023 18:15:10 +0000 Subject: [PATCH 3/4] rename module from elk to ccs --- .gitignore | 4 --- MANIFEST.in | 4 +-- README.md | 24 +++++++------- {elk => ccs}/__init__.py | 0 {elk => ccs}/__main__.py | 10 +++--- {elk => ccs}/debug_logging.py | 0 {elk => ccs}/evaluation/__init__.py | 0 {elk => ccs}/evaluation/evaluate.py | 6 ++-- {elk => ccs}/extraction/__init__.py | 0 {elk => ccs}/extraction/balanced_sampler.py | 0 {elk => ccs}/extraction/dataset_name.py | 0 {elk => ccs}/extraction/extraction.py | 0 {elk => ccs}/extraction/generator.py | 0 {elk => ccs}/extraction/prompt_loading.py | 0 {elk => ccs}/files.py | 10 +++--- {elk => ccs}/metrics/__init__.py | 0 {elk => ccs}/metrics/accuracy.py | 0 {elk => ccs}/metrics/calibration.py | 0 {elk => ccs}/metrics/eval.py | 0 {elk => ccs}/metrics/roc_auc.py | 0 {elk => ccs}/parsing.py | 0 {elk => ccs}/plotting/command.py | 0 {elk => ccs}/plotting/visualize.py | 0 {elk => ccs}/promptsource/LICENSE | 0 {elk => ccs}/promptsource/__init__.py | 0 {elk => ccs}/promptsource/templates.py | 0 .../Zaid/coqa_expanded/templates.yaml | 0 .../Zaid/quac_expanded/templates.yaml | 0 .../acronym_identification/templates.yaml | 0 .../templates.yaml | 0 .../templates.yaml | 0 .../templates.yaml | 0 .../adversarialQA/templates.yaml | 0 .../adversarial_qa/dbert/templates.yaml | 0 .../adversarial_qa/dbidaf/templates.yaml | 0 .../adversarial_qa/droberta/templates.yaml | 0 .../templates/aeslc/templates.yaml | 0 .../templates/ag_news/templates.yaml | 0 .../ai2_arc/ARC-Challenge/templates.yaml | 0 .../templates/ai2_arc/ARC-Easy/templates.yaml | 0 .../templates/amazon_polarity/templates.yaml | 0 .../amazon_reviews_multi/en/templates.yaml | 0 .../Wireless_v1_00/templates.yaml | 0 .../templates/ambig_qa/light/templates.yaml | 0 .../templates/anli/templates.yaml | 0 .../templates/app_reviews/templates.yaml | 0 .../templates/aqua_rat/raw/templates.yaml | 0 .../promptsource/templates/art/templates.yaml | 0 .../templates/asnq/templates.yaml | 0 .../templates/asset/ratings/templates.yaml | 0 .../asset/simplification/templates.yaml | 0 .../templates/banking77/templates.yaml | 0 .../templates/billsum/templates.yaml | 0 .../bing_coronavirus_query_set/templates.yaml | 0 .../templates/biosses/templates.yaml | 0 .../title_genre_classifiction/templates.yaml | 0 .../blended_skill_talk/templates.yaml | 0 .../templates/boolq_pt/templates.yaml | 0 .../templates/cbt/CN/templates.yaml | 0 .../templates/cbt/NE/templates.yaml | 0 .../templates/cbt/P/templates.yaml | 0 .../templates/cbt/V/templates.yaml | 0 .../templates/cbt/raw/templates.yaml | 0 .../templates/cc_news/templates.yaml | 0 .../christykoh/ag_news_pt/templates.yaml | 0 .../christykoh/boolq_pt/templates.yaml | 0 .../christykoh/imdb_ar/templates.yaml | 0 .../christykoh/imdb_es/templates.yaml | 0 .../christykoh/imdb_fr/templates.yaml | 0 .../christykoh/imdb_pt/templates.yaml | 0 .../christykoh/imdb_zh/templates.yaml | 0 .../templates/circa/templates.yaml | 0 .../templates/climate_fever/templates.yaml | 0 .../cnn_dailymail/3.0.0/templates.yaml | 0 .../templates/codah/codah/templates.yaml | 0 .../templates/codah/fold_0/templates.yaml | 0 .../templates/codah/fold_1/templates.yaml | 0 .../templates/codah/fold_2/templates.yaml | 0 .../templates/codah/fold_3/templates.yaml | 0 .../templates/codah/fold_4/templates.yaml | 0 .../templates.yaml | 0 .../templates/common_gen/templates.yaml | 0 .../templates/commonsense_qa/templates.yaml | 0 .../templates/conv_ai/templates.yaml | 0 .../templates/conv_ai_2/templates.yaml | 0 .../templates/conv_ai_3/templates.yaml | 0 .../templates/coqa/templates.yaml | 0 .../templates/cord19/metadata/templates.yaml | 0 .../templates/cos_e/v1.0/templates.yaml | 0 .../templates/cos_e/v1.11/templates.yaml | 0 .../templates/cosmos_qa/templates.yaml | 0 .../covid_qa_castorini/templates.yaml | 0 .../craffel/openai_lambada/templates.yaml | 0 .../craigslist_bargains/templates.yaml | 0 .../templates/crows_pairs/templates.yaml | 0 .../templates/dbpedia_14/templates.yaml | 0 .../discofuse/discofuse-sport/templates.yaml | 0 .../discofuse-wikipedia/templates.yaml | 0 .../discovery/discovery/templates.yaml | 0 .../templates/docred/templates.yaml | 0 .../templates/dream/templates.yaml | 0 .../templates/drop/templates.yaml | 0 .../duorc/ParaphraseRC/templates.yaml | 0 .../templates/duorc/SelfRC/templates.yaml | 0 .../templates/e2e_nlg_cleaned/templates.yaml | 0 .../templates.yaml | 0 .../promptsource/templates/emo/templates.yaml | 0 .../templates/emotion/templates.yaml | 0 .../enriched_web_nlg/en/templates.yaml | 0 .../templates/esnli/templates.yaml | 0 .../1.1/templates.yaml | 0 .../2.0/templates.yaml | 0 .../templates/fever/v1.0/templates.yaml | 0 .../templates/fever/v2.0/templates.yaml | 0 .../sentences_allagree/templates.yaml | 0 .../templates/freebase_qa/templates.yaml | 0 .../generated_reviews_enth/templates.yaml | 0 .../templates/gigaword/templates.yaml | 0 .../templates/glue/ax/templates.yaml | 0 .../templates/glue/cola/templates.yaml | 0 .../templates/glue/mnli/templates.yaml | 0 .../glue/mnli_matched/templates.yaml | 0 .../glue/mnli_mismatched/templates.yaml | 0 .../templates/glue/mrpc/templates.yaml | 0 .../templates/glue/qnli/templates.yaml | 0 .../templates/glue/qqp/templates.yaml | 0 .../templates/glue/rte/templates.yaml | 0 .../templates/glue/sst2/templates.yaml | 0 .../templates/glue/stsb/templates.yaml | 0 .../templates/glue/wnli/templates.yaml | 0 .../google_wellformed_query/templates.yaml | 0 .../templates/great_code/templates.yaml | 0 .../cross_genre_1/templates.yaml | 0 .../cross_topic_1/templates.yaml | 0 .../cross_topic_4/templates.yaml | 0 .../cross_topic_7/templates.yaml | 0 .../templates/gutenberg_time/templates.yaml | 0 .../templates/hans/templates.yaml | 0 .../templates/hate_speech18/templates.yaml | 0 .../templates/head_qa/en/templates.yaml | 0 .../templates/health_fact/templates.yaml | 0 .../templates/hellaswag/templates.yaml | 0 .../templates/hlgd/templates.yaml | 0 .../hotpot_qa/distractor/templates.yaml | 0 .../hotpot_qa/fullwiki/templates.yaml | 0 .../humicroedit/subtask-1/templates.yaml | 0 .../humicroedit/subtask-2/templates.yaml | 0 .../byarticle/templates.yaml | 0 .../bypublisher/templates.yaml | 0 .../templates/imdb/templates.yaml | 0 .../templates/jfleg/templates.yaml | 0 .../jigsaw_unintended_bias/templates.yaml | 0 .../templates/kelm/templates.yaml | 0 .../kilt_tasks/hotpotqa/templates.yaml | 0 .../templates/kilt_tasks/nq/templates.yaml | 0 .../templates/lama/trex/templates.yaml | 0 .../templates/lambada/templates.yaml | 0 .../lauritowal/redefine_math/templates.yaml | 0 .../templates/liar/templates.yaml | 0 .../templates/limit/templates.yaml | 0 .../algebra__linear_1d/templates.yaml | 0 .../templates.yaml | 0 .../algebra__linear_2d/templates.yaml | 0 .../templates.yaml | 0 .../templates/math_qa/templates.yaml | 0 .../templates/mc_taco/templates.yaml | 0 .../templates/mdd/task1_qa/templates.yaml | 0 .../templates/mdd/task2_recs/templates.yaml | 0 .../templates/mdd/task3_qarecs/templates.yaml | 0 .../templates/medal/templates.yaml | 0 .../medical_questions_pairs/templates.yaml | 0 .../meta_woz/dialogues/templates.yaml | 0 .../templates/mocha/templates.yaml | 0 .../templates/movie_rationales/templates.yaml | 0 .../templates/multi_news/templates.yaml | 0 .../templates/multi_nli/templates.yaml | 0 .../multi_x_science_sum/templates.yaml | 0 .../templates/mwsc/templates.yaml | 0 .../templates/narrativeqa/templates.yaml | 0 .../templates/ncbi_disease/templates.yaml | 0 .../evaluation_dataset/templates.yaml | 0 .../templates/newspop/templates.yaml | 0 .../nlu_evaluation_data/templates.yaml | 0 .../templates/nq_open/templates.yaml | 0 .../templates/numer_sense/templates.yaml | 0 .../templates/onestop_english/templates.yaml | 0 .../templates/openai_humaneval/templates.yaml | 0 .../openbookqa/additional/templates.yaml | 0 .../templates/openbookqa/main/templates.yaml | 0 .../templates/paws-x/en/templates.yaml | 0 .../paws/labeled_final/templates.yaml | 0 .../paws/labeled_swap/templates.yaml | 0 .../paws/unlabeled_final/templates.yaml | 0 .../templates/piqa/templates.yaml | 0 .../templates/poem_sentiment/templates.yaml | 0 .../pubmed_qa/pqa_labeled/templates.yaml | 0 .../templates/qa_srl/templates.yaml | 0 .../templates/qa_zre/templates.yaml | 0 .../templates/qasc/templates.yaml | 0 .../promptsource/templates/qed/templates.yaml | 0 .../templates/quac/templates.yaml | 0 .../templates/quail/templates.yaml | 0 .../templates/quarel/templates.yaml | 0 .../templates/quartz/templates.yaml | 0 .../templates/quora/templates.yaml | 0 .../templates/quoref/templates.yaml | 0 .../templates/race/all/templates.yaml | 0 .../templates/race/high/templates.yaml | 0 .../templates/race/middle/templates.yaml | 0 .../reaganjlee/boolq_ar/templates.yaml | 0 .../reaganjlee/boolq_es/templates.yaml | 0 .../reaganjlee/boolq_fr/templates.yaml | 0 .../reaganjlee/boolq_pt/templates.yaml | 0 .../reaganjlee/boolq_zh/templates.yaml | 0 .../reaganjlee/truthful_qa_mc/templates.yaml | 0 .../truthful_qa_mc_ar/templates.yaml | 0 .../truthful_qa_mc_es/templates.yaml | 0 .../truthful_qa_mc_fr/templates.yaml | 0 .../truthful_qa_mc_pt/templates.yaml | 0 .../truthful_qa_mc_zh/templates.yaml | 0 .../templates/riddle_sense/templates.yaml | 0 .../templates/ropes/templates.yaml | 0 .../templates/rotten_tomatoes/templates.yaml | 0 .../templates/samsum/templates.yaml | 0 .../scan/addprim_jump/templates.yaml | 0 .../scan/addprim_turn_left/templates.yaml | 0 .../templates/scan/filler_num0/templates.yaml | 0 .../templates/scan/filler_num1/templates.yaml | 0 .../templates/scan/filler_num2/templates.yaml | 0 .../templates/scan/filler_num3/templates.yaml | 0 .../templates/scan/length/templates.yaml | 0 .../templates/scan/simple/templates.yaml | 0 .../scan/template_around_right/templates.yaml | 0 .../template_jump_around_right/templates.yaml | 0 .../template_opposite_right/templates.yaml | 0 .../scan/template_right/templates.yaml | 0 .../templates/scicite/templates.yaml | 0 .../scientific_papers/arxiv/templates.yaml | 0 .../scientific_papers/pubmed/templates.yaml | 0 .../templates/sciq/templates.yaml | 0 .../scitail/snli_format/templates.yaml | 0 .../scitail/tsv_format/templates.yaml | 0 .../templates/scitldr/Abstract/templates.yaml | 0 .../answer_selection_analysis/templates.yaml | 0 .../sem_eval_2010_task_8/templates.yaml | 0 .../sem_eval_2014_task_1/templates.yaml | 0 .../templates/sent_comp/templates.yaml | 0 .../templates/sick/templates.yaml | 0 .../templates/sms_spam/templates.yaml | 0 .../snips_built_in_intents/templates.yaml | 0 .../templates/snli/templates.yaml | 0 .../templates/social_i_qa/templates.yaml | 0 .../templates/species_800/templates.yaml | 0 .../templates/squad/templates.yaml | 0 .../squad_adversarial/AddSent/templates.yaml | 0 .../templates/squad_v2/templates.yaml | 0 .../squadshifts/amazon/templates.yaml | 0 .../squadshifts/new_wiki/templates.yaml | 0 .../templates/squadshifts/nyt/templates.yaml | 0 .../templates/sst/default/templates.yaml | 0 .../templates/sst2/templates.yaml | 0 .../templates/story_cloze/2016/templates.yaml | 0 .../templates/stsb_multi_mt/en/templates.yaml | 0 .../templates/subjqa/books/templates.yaml | 0 .../subjqa/electronics/templates.yaml | 0 .../templates/subjqa/grocery/templates.yaml | 0 .../templates/subjqa/movies/templates.yaml | 0 .../subjqa/restaurants/templates.yaml | 0 .../subjqa/tripadvisor/templates.yaml | 0 .../templates/super_glue/axb/templates.yaml | 0 .../templates/super_glue/axg/templates.yaml | 0 .../templates/super_glue/boolq/templates.yaml | 0 .../templates/super_glue/cb/templates.yaml | 0 .../templates/super_glue/copa/templates.yaml | 0 .../super_glue/multirc/templates.yaml | 0 .../super_glue/record/templates.yaml | 0 .../templates/super_glue/rte/templates.yaml | 0 .../templates/super_glue/wic/templates.yaml | 0 .../super_glue/wsc.fixed/templates.yaml | 0 .../templates/swag/regular/templates.yaml | 0 .../tab_fact/tab_fact/templates.yaml | 0 .../templates/tmu_gfm_dataset/templates.yaml | 0 .../templates/trec/templates.yaml | 0 .../trivia_qa/unfiltered/templates.yaml | 0 .../templates/turk/templates.yaml | 0 .../templates/tweet_eval/emoji/templates.yaml | 0 .../tweet_eval/emotion/templates.yaml | 0 .../templates/tweet_eval/hate/templates.yaml | 0 .../templates/tweet_eval/irony/templates.yaml | 0 .../tweet_eval/offensive/templates.yaml | 0 .../tweet_eval/sentiment/templates.yaml | 0 .../tweet_eval/stance_abortion/templates.yaml | 0 .../tweet_eval/stance_atheism/templates.yaml | 0 .../tweet_eval/stance_climate/templates.yaml | 0 .../tweet_eval/stance_feminist/templates.yaml | 0 .../tweet_eval/stance_hillary/templates.yaml | 0 .../tydiqa/primary_task/templates.yaml | 0 .../tydiqa/secondary_task/templates.yaml | 0 .../templates/web_questions/templates.yaml | 0 .../templates/wiki_bio/templates.yaml | 0 .../templates/wiki_hop/masked/templates.yaml | 0 .../wiki_hop/original/templates.yaml | 0 .../templates/wiki_qa/templates.yaml | 0 .../templates/wiki_split/templates.yaml | 0 .../wino_bias/type1_anti/templates.yaml | 0 .../wino_bias/type1_pro/templates.yaml | 0 .../wino_bias/type2_anti/templates.yaml | 0 .../wino_bias/type2_pro/templates.yaml | 0 .../winograd_wsc/wsc273/templates.yaml | 0 .../winograd_wsc/wsc285/templates.yaml | 0 .../winogrande_debiased/templates.yaml | 0 .../winogrande/winogrande_l/templates.yaml | 0 .../winogrande/winogrande_m/templates.yaml | 0 .../winogrande/winogrande_s/templates.yaml | 0 .../winogrande/winogrande_xl/templates.yaml | 0 .../winogrande/winogrande_xs/templates.yaml | 0 .../templates/wiqa/templates.yaml | 0 .../templates/xnli/en/templates.yaml | 0 .../templates/xquad/xquad.en/templates.yaml | 0 .../templates/xquad_r/en/templates.yaml | 0 .../templates/xsum/templates.yaml | 0 .../templates/yahoo_answers_qa/templates.yaml | 0 .../yahoo_answers_topics/templates.yaml | 0 .../templates/yelp_polarity/templates.yaml | 0 .../templates/yelp_review_full/templates.yaml | 0 .../templates/zest/templates.yaml | 0 {elk => ccs}/resources/adjectives.json | 0 {elk => ccs}/resources/names.json | 0 {elk => ccs}/run.py | 6 ++-- {elk => ccs}/training/__init__.py | 0 {elk => ccs}/training/burns_norm.py | 0 {elk => ccs}/training/ccs_reporter.py | 2 +- {elk => ccs}/training/classifier.py | 0 {elk => ccs}/training/common.py | 0 {elk => ccs}/training/eigen_reporter.py | 0 {elk => ccs}/training/losses.py | 0 {elk => ccs}/training/platt_scaling.py | 0 {elk => ccs}/training/supervised.py | 0 {elk => ccs}/training/sweep.py | 0 {elk => ccs}/training/train.py | 0 {elk => ccs}/truncated_eigh.py | 0 {elk => ccs}/utils/__init__.py | 0 {elk => ccs}/utils/constants.py | 0 {elk => ccs}/utils/data_utils.py | 0 {elk => ccs}/utils/gpu_utils.py | 0 {elk => ccs}/utils/hf_utils.py | 0 {elk => ccs}/utils/math_util.py | 0 {elk => ccs}/utils/pretty.py | 0 {elk => ccs}/utils/tree_utils.py | 0 {elk => ccs}/utils/typing.py | 0 pyproject.toml | 12 +++---- tests/test_burns_norm.py | 2 +- tests/test_classifier.py | 2 +- tests/test_eigen_reporter.py | 4 +-- tests/test_load_prompts.py | 4 +-- tests/test_math.py | 2 +- tests/test_metrics.py | 2 +- tests/test_samplers.py | 4 +-- tests/test_smoke_elicit.py | 6 ++-- tests/test_smoke_eval.py | 8 ++--- tests/test_truncated_eigh.py | 2 +- tests/test_viz.py | 31 ------------------- 362 files changed, 55 insertions(+), 90 deletions(-) rename {elk => ccs}/__init__.py (100%) rename {elk => ccs}/__main__.py (71%) rename {elk => ccs}/debug_logging.py (100%) rename {elk => ccs}/evaluation/__init__.py (100%) rename {elk => ccs}/evaluation/evaluate.py (95%) rename {elk => ccs}/extraction/__init__.py (100%) rename {elk => ccs}/extraction/balanced_sampler.py (100%) rename {elk => ccs}/extraction/dataset_name.py (100%) rename {elk => ccs}/extraction/extraction.py (100%) rename {elk => ccs}/extraction/generator.py (100%) rename {elk => ccs}/extraction/prompt_loading.py (100%) rename {elk => ccs}/files.py (84%) rename {elk => ccs}/metrics/__init__.py (100%) rename {elk => ccs}/metrics/accuracy.py (100%) rename {elk => ccs}/metrics/calibration.py (100%) rename {elk => ccs}/metrics/eval.py (100%) rename {elk => ccs}/metrics/roc_auc.py (100%) rename {elk => ccs}/parsing.py (100%) rename {elk => ccs}/plotting/command.py (100%) rename {elk => ccs}/plotting/visualize.py (100%) rename {elk => ccs}/promptsource/LICENSE (100%) rename {elk => ccs}/promptsource/__init__.py (100%) rename {elk => ccs}/promptsource/templates.py (100%) rename {elk => ccs}/promptsource/templates/Zaid/coqa_expanded/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/Zaid/quac_expanded/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/acronym_identification/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/adversarial_qa/dbert/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/adversarial_qa/dbidaf/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/adversarial_qa/droberta/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/aeslc/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ag_news/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/amazon_polarity/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/amazon_reviews_multi/en/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ambig_qa/light/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/anli/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/app_reviews/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/aqua_rat/raw/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/art/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/asnq/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/asset/ratings/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/asset/simplification/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/banking77/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/billsum/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/bing_coronavirus_query_set/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/biosses/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/blended_skill_talk/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/boolq_pt/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cbt/CN/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cbt/NE/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cbt/P/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cbt/V/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cbt/raw/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cc_news/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/christykoh/ag_news_pt/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/christykoh/boolq_pt/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/christykoh/imdb_ar/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/christykoh/imdb_es/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/christykoh/imdb_fr/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/christykoh/imdb_pt/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/christykoh/imdb_zh/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/circa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/climate_fever/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/codah/codah/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/codah/fold_0/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/codah/fold_1/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/codah/fold_2/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/codah/fold_3/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/codah/fold_4/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/common_gen/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/commonsense_qa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/conv_ai/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/conv_ai_2/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/conv_ai_3/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/coqa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cord19/metadata/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cos_e/v1.0/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cos_e/v1.11/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/cosmos_qa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/covid_qa_castorini/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/craffel/openai_lambada/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/craigslist_bargains/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/crows_pairs/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/dbpedia_14/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/discofuse/discofuse-sport/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/discovery/discovery/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/docred/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/dream/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/drop/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/duorc/ParaphraseRC/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/duorc/SelfRC/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/e2e_nlg_cleaned/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/emo/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/emotion/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/enriched_web_nlg/en/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/esnli/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/fever/v1.0/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/fever/v2.0/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/freebase_qa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/generated_reviews_enth/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/gigaword/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/ax/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/cola/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/mnli/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/mnli_matched/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/mnli_mismatched/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/mrpc/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/qnli/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/qqp/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/rte/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/sst2/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/stsb/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/glue/wnli/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/google_wellformed_query/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/great_code/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/gutenberg_time/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/hans/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/hate_speech18/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/head_qa/en/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/health_fact/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/hellaswag/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/hlgd/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/hotpot_qa/distractor/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/hotpot_qa/fullwiki/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/humicroedit/subtask-1/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/humicroedit/subtask-2/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/imdb/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/jfleg/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/jigsaw_unintended_bias/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/kelm/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/kilt_tasks/nq/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/lama/trex/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/lambada/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/lauritowal/redefine_math/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/liar/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/limit/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/math_qa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/mc_taco/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/mdd/task1_qa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/mdd/task2_recs/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/mdd/task3_qarecs/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/medal/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/medical_questions_pairs/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/meta_woz/dialogues/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/mocha/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/movie_rationales/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/multi_news/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/multi_nli/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/multi_x_science_sum/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/mwsc/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/narrativeqa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ncbi_disease/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/newspop/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/nlu_evaluation_data/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/nq_open/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/numer_sense/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/onestop_english/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/openai_humaneval/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/openbookqa/additional/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/openbookqa/main/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/paws-x/en/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/paws/labeled_final/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/paws/labeled_swap/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/paws/unlabeled_final/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/piqa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/poem_sentiment/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/qa_srl/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/qa_zre/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/qasc/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/qed/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/quac/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/quail/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/quarel/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/quartz/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/quora/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/quoref/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/race/all/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/race/high/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/race/middle/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_ar/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_es/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_fr/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_pt/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_zh/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/riddle_sense/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/ropes/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/rotten_tomatoes/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/samsum/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/addprim_jump/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/addprim_turn_left/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/filler_num0/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/filler_num1/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/filler_num2/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/filler_num3/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/length/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/simple/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/template_around_right/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/template_jump_around_right/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/template_opposite_right/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scan/template_right/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scicite/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scientific_papers/arxiv/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scientific_papers/pubmed/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/sciq/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scitail/snli_format/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scitail/tsv_format/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/scitldr/Abstract/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/selqa/answer_selection_analysis/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/sem_eval_2010_task_8/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/sem_eval_2014_task_1/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/sent_comp/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/sick/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/sms_spam/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/snips_built_in_intents/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/snli/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/social_i_qa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/species_800/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/squad/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/squad_adversarial/AddSent/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/squad_v2/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/squadshifts/amazon/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/squadshifts/new_wiki/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/squadshifts/nyt/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/sst/default/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/sst2/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/story_cloze/2016/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/stsb_multi_mt/en/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/subjqa/books/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/subjqa/electronics/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/subjqa/grocery/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/subjqa/movies/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/subjqa/restaurants/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/subjqa/tripadvisor/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/axb/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/axg/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/boolq/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/cb/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/copa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/multirc/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/record/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/rte/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/wic/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/super_glue/wsc.fixed/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/swag/regular/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tab_fact/tab_fact/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tmu_gfm_dataset/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/trec/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/trivia_qa/unfiltered/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/turk/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/emoji/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/emotion/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/hate/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/irony/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/offensive/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/sentiment/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/stance_abortion/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/stance_atheism/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/stance_climate/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/stance_feminist/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tweet_eval/stance_hillary/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tydiqa/primary_task/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/tydiqa/secondary_task/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/web_questions/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wiki_bio/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wiki_hop/masked/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wiki_hop/original/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wiki_qa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wiki_split/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wino_bias/type1_anti/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wino_bias/type1_pro/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wino_bias/type2_anti/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wino_bias/type2_pro/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/winograd_wsc/wsc273/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/winograd_wsc/wsc285/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/winogrande/winogrande_debiased/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/winogrande/winogrande_l/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/winogrande/winogrande_m/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/winogrande/winogrande_s/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/winogrande/winogrande_xl/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/winogrande/winogrande_xs/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/wiqa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/xnli/en/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/xquad/xquad.en/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/xquad_r/en/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/xsum/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/yahoo_answers_qa/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/yahoo_answers_topics/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/yelp_polarity/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/yelp_review_full/templates.yaml (100%) rename {elk => ccs}/promptsource/templates/zest/templates.yaml (100%) rename {elk => ccs}/resources/adjectives.json (100%) rename {elk => ccs}/resources/names.json (100%) rename {elk => ccs}/run.py (97%) rename {elk => ccs}/training/__init__.py (100%) rename {elk => ccs}/training/burns_norm.py (100%) rename {elk => ccs}/training/ccs_reporter.py (99%) rename {elk => ccs}/training/classifier.py (100%) rename {elk => ccs}/training/common.py (100%) rename {elk => ccs}/training/eigen_reporter.py (100%) rename {elk => ccs}/training/losses.py (100%) rename {elk => ccs}/training/platt_scaling.py (100%) rename {elk => ccs}/training/supervised.py (100%) rename {elk => ccs}/training/sweep.py (100%) rename {elk => ccs}/training/train.py (100%) rename {elk => ccs}/truncated_eigh.py (100%) rename {elk => ccs}/utils/__init__.py (100%) rename {elk => ccs}/utils/constants.py (100%) rename {elk => ccs}/utils/data_utils.py (100%) rename {elk => ccs}/utils/gpu_utils.py (100%) rename {elk => ccs}/utils/hf_utils.py (100%) rename {elk => ccs}/utils/math_util.py (100%) rename {elk => ccs}/utils/pretty.py (100%) rename {elk => ccs}/utils/tree_utils.py (100%) rename {elk => ccs}/utils/typing.py (100%) delete mode 100644 tests/test_viz.py diff --git a/.gitignore b/.gitignore index 6c29ee26..2704fbd8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,6 @@ *.csv *.npy -elk/models/* -elk/trained/* -nohup.out .idea -*.pkl # scripts for experiments in progress my_*.sh diff --git a/MANIFEST.in b/MANIFEST.in index 416520ef..d68d1589 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -recursive-include elk/promptsource/templates * -recursive-include elk/resources * +recursive-include ccs/promptsource/templates * +recursive-include ccs/resources * diff --git a/README.md b/README.md index fa74d900..10b8166a 100644 --- a/README.md +++ b/README.md @@ -21,16 +21,16 @@ Our code is based on [PyTorch](http://pytorch.org) and [Huggingface Transformers](https://huggingface.co/docs/transformers/index). We test the code on Python 3.10 and 3.11. -First install the package with `pip install -e .` in the root directory, or `pip install eleuther-elk` to install from PyPi. Use `pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies. +First install the package with `pip install -e .` in the root directory. Use `pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies. To fit reporters for the HuggingFace model `model` and dataset `dataset`, just run: ```bash -elk elicit microsoft/deberta-v2-xxlarge-mnli imdb +ccs elicit microsoft/deberta-v2-xxlarge-mnli imdb ``` This will automatically download the model and dataset, run the model and extract the relevant representations if they -aren't cached on disk, fit reporters on them, and save the reporter checkpoints to the `elk-reporters` folder in your +aren't cached on disk, fit reporters on them, and save the reporter checkpoints to the `ccs-reporters` folder in your home directory. It will also evaluate the reporter classification performance on a held out test set and save it to a CSV file in the same folder. @@ -38,35 +38,35 @@ The following will generate a CCS (Contrast Consistent Search) reporter instead default. ```bash -elk elicit microsoft/deberta-v2-xxlarge-mnli imdb --net ccs +ccs elicit microsoft/deberta-v2-xxlarge-mnli imdb --net ccs ``` The following command will evaluate the probe from the run naughty-northcutt on the hidden states extracted from the model deberta-v2-xxlarge-mnli for the imdb dataset. It will result in an `eval.csv` and `cfg.yaml` file, which are -stored under a subfolder in `elk-reporters/naughty-northcutt/transfer_eval`. +stored under a subfolder in `ccs-reporters/naughty-northcutt/transfer_eval`. ```bash -elk eval naughty-northcutt microsoft/deberta-v2-xxlarge-mnli imdb +ccs eval naughty-northcutt microsoft/deberta-v2-xxlarge-mnli imdb ``` The following runs `elicit` on the Cartesian product of the listed models and datasets, storing it in a special folder -ELK_DIR/sweeps/. Moreover, `--add_pooled` adds an additional dataset that pools all of the datasets +CCS_DIR/sweeps/. Moreover, `--add_pooled` adds an additional dataset that pools all of the datasets together. You can also add a `--visualize` flag to visualize the results of the sweep. ```bash -elk sweep --models gpt2-{medium,large,xl} --datasets imdb amazon_polarity --add_pooled +ccs sweep --models gpt2-{medium,large,xl} --datasets imdb amazon_polarity --add_pooled ``` -If you just do `elk plot`, it will plot the results from the most recent sweep. +If you just do `ccs plot`, it will plot the results from the most recent sweep. If you want to plot a specific sweep, you can do so with: ```bash -elk plot {sweep_name} +ccs plot {sweep_name} ``` ## Caching -The hidden states resulting from `elk elicit` are cached as a HuggingFace dataset to avoid having to recompute them +The hidden states resulting from `ccs elicit` are cached as a HuggingFace dataset to avoid having to recompute them every time we want to train a probe. The cache is stored in the same place as all other HuggingFace datasets, which is usually `~/.cache/huggingface/datasets`. @@ -81,7 +81,7 @@ Use `pip install pre-commit && pre-commit install` in the root folder before you https://img.shields.io/static/v1?label=Remote%20-%20Containers&message=Open&color=blue&logo=visualstudiocode ) ]( -https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/EleutherAI/elk +https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/EleutherAI/ccs ) ### Run tests diff --git a/elk/__init__.py b/ccs/__init__.py similarity index 100% rename from elk/__init__.py rename to ccs/__init__.py diff --git a/elk/__main__.py b/ccs/__main__.py similarity index 71% rename from elk/__main__.py rename to ccs/__main__.py index 86e5087a..b1caef9b 100644 --- a/elk/__main__.py +++ b/ccs/__main__.py @@ -1,13 +1,13 @@ -"""Main entry point for `elk`.""" +"""Main entry point for `ccs`.""" from dataclasses import dataclass from simple_parsing import ArgumentParser -from elk.evaluation.evaluate import Eval -from elk.plotting.command import Plot -from elk.training.sweep import Sweep -from elk.training.train import Elicit +from ccs.evaluation.evaluate import Eval +from ccs.plotting.command import Plot +from ccs.training.sweep import Sweep +from ccs.training.train import Elicit @dataclass diff --git a/elk/debug_logging.py b/ccs/debug_logging.py similarity index 100% rename from elk/debug_logging.py rename to ccs/debug_logging.py diff --git a/elk/evaluation/__init__.py b/ccs/evaluation/__init__.py similarity index 100% rename from elk/evaluation/__init__.py rename to ccs/evaluation/__init__.py diff --git a/elk/evaluation/evaluate.py b/ccs/evaluation/evaluate.py similarity index 95% rename from elk/evaluation/evaluate.py rename to ccs/evaluation/evaluate.py index 4e6ec59f..53df3715 100644 --- a/elk/evaluation/evaluate.py +++ b/ccs/evaluation/evaluate.py @@ -6,7 +6,7 @@ import torch from simple_parsing.helpers import field -from ..files import elk_reporter_dir +from ..files import ccs_reporter_dir from ..metrics import evaluate_preds from ..run import Run from ..utils import Color @@ -22,7 +22,7 @@ class Eval(Run): def __post_init__(self): # Set our output directory before super().execute() does if not self.out_dir: - root = elk_reporter_dir() / self.source + root = ccs_reporter_dir() / self.source self.out_dir = root / "transfer" / "+".join(self.data.datasets) def execute(self, highlight_color: Color = "cyan"): @@ -36,7 +36,7 @@ def apply_to_layer( device = self.get_device(devices, world_size) val_output = self.prepare_data(device, layer, "val") - experiment_dir = elk_reporter_dir() / self.source + experiment_dir = ccs_reporter_dir() / self.source reporter_path = experiment_dir / "reporters" / f"layer_{layer}.pt" reporter = torch.load(reporter_path, map_location=device) diff --git a/elk/extraction/__init__.py b/ccs/extraction/__init__.py similarity index 100% rename from elk/extraction/__init__.py rename to ccs/extraction/__init__.py diff --git a/elk/extraction/balanced_sampler.py b/ccs/extraction/balanced_sampler.py similarity index 100% rename from elk/extraction/balanced_sampler.py rename to ccs/extraction/balanced_sampler.py diff --git a/elk/extraction/dataset_name.py b/ccs/extraction/dataset_name.py similarity index 100% rename from elk/extraction/dataset_name.py rename to ccs/extraction/dataset_name.py diff --git a/elk/extraction/extraction.py b/ccs/extraction/extraction.py similarity index 100% rename from elk/extraction/extraction.py rename to ccs/extraction/extraction.py diff --git a/elk/extraction/generator.py b/ccs/extraction/generator.py similarity index 100% rename from elk/extraction/generator.py rename to ccs/extraction/generator.py diff --git a/elk/extraction/prompt_loading.py b/ccs/extraction/prompt_loading.py similarity index 100% rename from elk/extraction/prompt_loading.py rename to ccs/extraction/prompt_loading.py diff --git a/elk/files.py b/ccs/files.py similarity index 84% rename from elk/files.py rename to ccs/files.py index 8546d4ab..96d5dd88 100644 --- a/elk/files.py +++ b/ccs/files.py @@ -8,14 +8,14 @@ def sweeps_dir() -> Path: """Return the directory where sweeps are stored.""" - return elk_reporter_dir() / "sweeps" + return ccs_reporter_dir() / "sweeps" -def elk_reporter_dir() -> Path: +def ccs_reporter_dir() -> Path: """Return the directory where reporter checkpoints and logs are stored.""" - env_dir = os.environ.get("ELK_DIR", None) + env_dir = os.environ.get("CCS_DIR", None) if env_dir is None: - log_dir = Path.home() / "elk-reporters" + log_dir = Path.home() / "ccs-reporters" else: log_dir = Path(env_dir) @@ -47,4 +47,4 @@ def memorably_named_dir(parent: Path): def transfer_eval_directory(source: str) -> Path: """Return the directory where transfer evals are stored.""" - return elk_reporter_dir() / source / "transfer_eval" + return ccs_reporter_dir() / source / "transfer_eval" diff --git a/elk/metrics/__init__.py b/ccs/metrics/__init__.py similarity index 100% rename from elk/metrics/__init__.py rename to ccs/metrics/__init__.py diff --git a/elk/metrics/accuracy.py b/ccs/metrics/accuracy.py similarity index 100% rename from elk/metrics/accuracy.py rename to ccs/metrics/accuracy.py diff --git a/elk/metrics/calibration.py b/ccs/metrics/calibration.py similarity index 100% rename from elk/metrics/calibration.py rename to ccs/metrics/calibration.py diff --git a/elk/metrics/eval.py b/ccs/metrics/eval.py similarity index 100% rename from elk/metrics/eval.py rename to ccs/metrics/eval.py diff --git a/elk/metrics/roc_auc.py b/ccs/metrics/roc_auc.py similarity index 100% rename from elk/metrics/roc_auc.py rename to ccs/metrics/roc_auc.py diff --git a/elk/parsing.py b/ccs/parsing.py similarity index 100% rename from elk/parsing.py rename to ccs/parsing.py diff --git a/elk/plotting/command.py b/ccs/plotting/command.py similarity index 100% rename from elk/plotting/command.py rename to ccs/plotting/command.py diff --git a/elk/plotting/visualize.py b/ccs/plotting/visualize.py similarity index 100% rename from elk/plotting/visualize.py rename to ccs/plotting/visualize.py diff --git a/elk/promptsource/LICENSE b/ccs/promptsource/LICENSE similarity index 100% rename from elk/promptsource/LICENSE rename to ccs/promptsource/LICENSE diff --git a/elk/promptsource/__init__.py b/ccs/promptsource/__init__.py similarity index 100% rename from elk/promptsource/__init__.py rename to ccs/promptsource/__init__.py diff --git a/elk/promptsource/templates.py b/ccs/promptsource/templates.py similarity index 100% rename from elk/promptsource/templates.py rename to ccs/promptsource/templates.py diff --git a/elk/promptsource/templates/Zaid/coqa_expanded/templates.yaml b/ccs/promptsource/templates/Zaid/coqa_expanded/templates.yaml similarity index 100% rename from elk/promptsource/templates/Zaid/coqa_expanded/templates.yaml rename to ccs/promptsource/templates/Zaid/coqa_expanded/templates.yaml diff --git a/elk/promptsource/templates/Zaid/quac_expanded/templates.yaml b/ccs/promptsource/templates/Zaid/quac_expanded/templates.yaml similarity index 100% rename from elk/promptsource/templates/Zaid/quac_expanded/templates.yaml rename to ccs/promptsource/templates/Zaid/quac_expanded/templates.yaml diff --git a/elk/promptsource/templates/acronym_identification/templates.yaml b/ccs/promptsource/templates/acronym_identification/templates.yaml similarity index 100% rename from elk/promptsource/templates/acronym_identification/templates.yaml rename to ccs/promptsource/templates/acronym_identification/templates.yaml diff --git a/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml b/ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml similarity index 100% rename from elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml rename to ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml diff --git a/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml b/ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml similarity index 100% rename from elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml rename to ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml diff --git a/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml b/ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml similarity index 100% rename from elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml rename to ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml diff --git a/elk/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml b/ccs/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml similarity index 100% rename from elk/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml rename to ccs/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml diff --git a/elk/promptsource/templates/adversarial_qa/dbert/templates.yaml b/ccs/promptsource/templates/adversarial_qa/dbert/templates.yaml similarity index 100% rename from elk/promptsource/templates/adversarial_qa/dbert/templates.yaml rename to ccs/promptsource/templates/adversarial_qa/dbert/templates.yaml diff --git a/elk/promptsource/templates/adversarial_qa/dbidaf/templates.yaml b/ccs/promptsource/templates/adversarial_qa/dbidaf/templates.yaml similarity index 100% rename from elk/promptsource/templates/adversarial_qa/dbidaf/templates.yaml rename to ccs/promptsource/templates/adversarial_qa/dbidaf/templates.yaml diff --git a/elk/promptsource/templates/adversarial_qa/droberta/templates.yaml b/ccs/promptsource/templates/adversarial_qa/droberta/templates.yaml similarity index 100% rename from elk/promptsource/templates/adversarial_qa/droberta/templates.yaml rename to ccs/promptsource/templates/adversarial_qa/droberta/templates.yaml diff --git a/elk/promptsource/templates/aeslc/templates.yaml b/ccs/promptsource/templates/aeslc/templates.yaml similarity index 100% rename from elk/promptsource/templates/aeslc/templates.yaml rename to ccs/promptsource/templates/aeslc/templates.yaml diff --git a/elk/promptsource/templates/ag_news/templates.yaml b/ccs/promptsource/templates/ag_news/templates.yaml similarity index 100% rename from elk/promptsource/templates/ag_news/templates.yaml rename to ccs/promptsource/templates/ag_news/templates.yaml diff --git a/elk/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml b/ccs/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml similarity index 100% rename from elk/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml rename to ccs/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml diff --git a/elk/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml b/ccs/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml similarity index 100% rename from elk/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml rename to ccs/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml diff --git a/elk/promptsource/templates/amazon_polarity/templates.yaml b/ccs/promptsource/templates/amazon_polarity/templates.yaml similarity index 100% rename from elk/promptsource/templates/amazon_polarity/templates.yaml rename to ccs/promptsource/templates/amazon_polarity/templates.yaml diff --git a/elk/promptsource/templates/amazon_reviews_multi/en/templates.yaml b/ccs/promptsource/templates/amazon_reviews_multi/en/templates.yaml similarity index 100% rename from elk/promptsource/templates/amazon_reviews_multi/en/templates.yaml rename to ccs/promptsource/templates/amazon_reviews_multi/en/templates.yaml diff --git a/elk/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml b/ccs/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml similarity index 100% rename from elk/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml rename to ccs/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml diff --git a/elk/promptsource/templates/ambig_qa/light/templates.yaml b/ccs/promptsource/templates/ambig_qa/light/templates.yaml similarity index 100% rename from elk/promptsource/templates/ambig_qa/light/templates.yaml rename to ccs/promptsource/templates/ambig_qa/light/templates.yaml diff --git a/elk/promptsource/templates/anli/templates.yaml b/ccs/promptsource/templates/anli/templates.yaml similarity index 100% rename from elk/promptsource/templates/anli/templates.yaml rename to ccs/promptsource/templates/anli/templates.yaml diff --git a/elk/promptsource/templates/app_reviews/templates.yaml b/ccs/promptsource/templates/app_reviews/templates.yaml similarity index 100% rename from elk/promptsource/templates/app_reviews/templates.yaml rename to ccs/promptsource/templates/app_reviews/templates.yaml diff --git a/elk/promptsource/templates/aqua_rat/raw/templates.yaml b/ccs/promptsource/templates/aqua_rat/raw/templates.yaml similarity index 100% rename from elk/promptsource/templates/aqua_rat/raw/templates.yaml rename to ccs/promptsource/templates/aqua_rat/raw/templates.yaml diff --git a/elk/promptsource/templates/art/templates.yaml b/ccs/promptsource/templates/art/templates.yaml similarity index 100% rename from elk/promptsource/templates/art/templates.yaml rename to ccs/promptsource/templates/art/templates.yaml diff --git a/elk/promptsource/templates/asnq/templates.yaml b/ccs/promptsource/templates/asnq/templates.yaml similarity index 100% rename from elk/promptsource/templates/asnq/templates.yaml rename to ccs/promptsource/templates/asnq/templates.yaml diff --git a/elk/promptsource/templates/asset/ratings/templates.yaml b/ccs/promptsource/templates/asset/ratings/templates.yaml similarity index 100% rename from elk/promptsource/templates/asset/ratings/templates.yaml rename to ccs/promptsource/templates/asset/ratings/templates.yaml diff --git a/elk/promptsource/templates/asset/simplification/templates.yaml b/ccs/promptsource/templates/asset/simplification/templates.yaml similarity index 100% rename from elk/promptsource/templates/asset/simplification/templates.yaml rename to ccs/promptsource/templates/asset/simplification/templates.yaml diff --git a/elk/promptsource/templates/banking77/templates.yaml b/ccs/promptsource/templates/banking77/templates.yaml similarity index 100% rename from elk/promptsource/templates/banking77/templates.yaml rename to ccs/promptsource/templates/banking77/templates.yaml diff --git a/elk/promptsource/templates/billsum/templates.yaml b/ccs/promptsource/templates/billsum/templates.yaml similarity index 100% rename from elk/promptsource/templates/billsum/templates.yaml rename to ccs/promptsource/templates/billsum/templates.yaml diff --git a/elk/promptsource/templates/bing_coronavirus_query_set/templates.yaml b/ccs/promptsource/templates/bing_coronavirus_query_set/templates.yaml similarity index 100% rename from elk/promptsource/templates/bing_coronavirus_query_set/templates.yaml rename to ccs/promptsource/templates/bing_coronavirus_query_set/templates.yaml diff --git a/elk/promptsource/templates/biosses/templates.yaml b/ccs/promptsource/templates/biosses/templates.yaml similarity index 100% rename from elk/promptsource/templates/biosses/templates.yaml rename to ccs/promptsource/templates/biosses/templates.yaml diff --git a/elk/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml b/ccs/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml similarity index 100% rename from elk/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml rename to ccs/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml diff --git a/elk/promptsource/templates/blended_skill_talk/templates.yaml b/ccs/promptsource/templates/blended_skill_talk/templates.yaml similarity index 100% rename from elk/promptsource/templates/blended_skill_talk/templates.yaml rename to ccs/promptsource/templates/blended_skill_talk/templates.yaml diff --git a/elk/promptsource/templates/boolq_pt/templates.yaml b/ccs/promptsource/templates/boolq_pt/templates.yaml similarity index 100% rename from elk/promptsource/templates/boolq_pt/templates.yaml rename to ccs/promptsource/templates/boolq_pt/templates.yaml diff --git a/elk/promptsource/templates/cbt/CN/templates.yaml b/ccs/promptsource/templates/cbt/CN/templates.yaml similarity index 100% rename from elk/promptsource/templates/cbt/CN/templates.yaml rename to ccs/promptsource/templates/cbt/CN/templates.yaml diff --git a/elk/promptsource/templates/cbt/NE/templates.yaml b/ccs/promptsource/templates/cbt/NE/templates.yaml similarity index 100% rename from elk/promptsource/templates/cbt/NE/templates.yaml rename to ccs/promptsource/templates/cbt/NE/templates.yaml diff --git a/elk/promptsource/templates/cbt/P/templates.yaml b/ccs/promptsource/templates/cbt/P/templates.yaml similarity index 100% rename from elk/promptsource/templates/cbt/P/templates.yaml rename to ccs/promptsource/templates/cbt/P/templates.yaml diff --git a/elk/promptsource/templates/cbt/V/templates.yaml b/ccs/promptsource/templates/cbt/V/templates.yaml similarity index 100% rename from elk/promptsource/templates/cbt/V/templates.yaml rename to ccs/promptsource/templates/cbt/V/templates.yaml diff --git a/elk/promptsource/templates/cbt/raw/templates.yaml b/ccs/promptsource/templates/cbt/raw/templates.yaml similarity index 100% rename from elk/promptsource/templates/cbt/raw/templates.yaml rename to ccs/promptsource/templates/cbt/raw/templates.yaml diff --git a/elk/promptsource/templates/cc_news/templates.yaml b/ccs/promptsource/templates/cc_news/templates.yaml similarity index 100% rename from elk/promptsource/templates/cc_news/templates.yaml rename to ccs/promptsource/templates/cc_news/templates.yaml diff --git a/elk/promptsource/templates/christykoh/ag_news_pt/templates.yaml b/ccs/promptsource/templates/christykoh/ag_news_pt/templates.yaml similarity index 100% rename from elk/promptsource/templates/christykoh/ag_news_pt/templates.yaml rename to ccs/promptsource/templates/christykoh/ag_news_pt/templates.yaml diff --git a/elk/promptsource/templates/christykoh/boolq_pt/templates.yaml b/ccs/promptsource/templates/christykoh/boolq_pt/templates.yaml similarity index 100% rename from elk/promptsource/templates/christykoh/boolq_pt/templates.yaml rename to ccs/promptsource/templates/christykoh/boolq_pt/templates.yaml diff --git a/elk/promptsource/templates/christykoh/imdb_ar/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_ar/templates.yaml similarity index 100% rename from elk/promptsource/templates/christykoh/imdb_ar/templates.yaml rename to ccs/promptsource/templates/christykoh/imdb_ar/templates.yaml diff --git a/elk/promptsource/templates/christykoh/imdb_es/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_es/templates.yaml similarity index 100% rename from elk/promptsource/templates/christykoh/imdb_es/templates.yaml rename to ccs/promptsource/templates/christykoh/imdb_es/templates.yaml diff --git a/elk/promptsource/templates/christykoh/imdb_fr/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_fr/templates.yaml similarity index 100% rename from elk/promptsource/templates/christykoh/imdb_fr/templates.yaml rename to ccs/promptsource/templates/christykoh/imdb_fr/templates.yaml diff --git a/elk/promptsource/templates/christykoh/imdb_pt/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_pt/templates.yaml similarity index 100% rename from elk/promptsource/templates/christykoh/imdb_pt/templates.yaml rename to ccs/promptsource/templates/christykoh/imdb_pt/templates.yaml diff --git a/elk/promptsource/templates/christykoh/imdb_zh/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_zh/templates.yaml similarity index 100% rename from elk/promptsource/templates/christykoh/imdb_zh/templates.yaml rename to ccs/promptsource/templates/christykoh/imdb_zh/templates.yaml diff --git a/elk/promptsource/templates/circa/templates.yaml b/ccs/promptsource/templates/circa/templates.yaml similarity index 100% rename from elk/promptsource/templates/circa/templates.yaml rename to ccs/promptsource/templates/circa/templates.yaml diff --git a/elk/promptsource/templates/climate_fever/templates.yaml b/ccs/promptsource/templates/climate_fever/templates.yaml similarity index 100% rename from elk/promptsource/templates/climate_fever/templates.yaml rename to ccs/promptsource/templates/climate_fever/templates.yaml diff --git a/elk/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml b/ccs/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml similarity index 100% rename from elk/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml rename to ccs/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml diff --git a/elk/promptsource/templates/codah/codah/templates.yaml b/ccs/promptsource/templates/codah/codah/templates.yaml similarity index 100% rename from elk/promptsource/templates/codah/codah/templates.yaml rename to ccs/promptsource/templates/codah/codah/templates.yaml diff --git a/elk/promptsource/templates/codah/fold_0/templates.yaml b/ccs/promptsource/templates/codah/fold_0/templates.yaml similarity index 100% rename from elk/promptsource/templates/codah/fold_0/templates.yaml rename to ccs/promptsource/templates/codah/fold_0/templates.yaml diff --git a/elk/promptsource/templates/codah/fold_1/templates.yaml b/ccs/promptsource/templates/codah/fold_1/templates.yaml similarity index 100% rename from elk/promptsource/templates/codah/fold_1/templates.yaml rename to ccs/promptsource/templates/codah/fold_1/templates.yaml diff --git a/elk/promptsource/templates/codah/fold_2/templates.yaml b/ccs/promptsource/templates/codah/fold_2/templates.yaml similarity index 100% rename from elk/promptsource/templates/codah/fold_2/templates.yaml rename to ccs/promptsource/templates/codah/fold_2/templates.yaml diff --git a/elk/promptsource/templates/codah/fold_3/templates.yaml b/ccs/promptsource/templates/codah/fold_3/templates.yaml similarity index 100% rename from elk/promptsource/templates/codah/fold_3/templates.yaml rename to ccs/promptsource/templates/codah/fold_3/templates.yaml diff --git a/elk/promptsource/templates/codah/fold_4/templates.yaml b/ccs/promptsource/templates/codah/fold_4/templates.yaml similarity index 100% rename from elk/promptsource/templates/codah/fold_4/templates.yaml rename to ccs/promptsource/templates/codah/fold_4/templates.yaml diff --git a/elk/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml b/ccs/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml similarity index 100% rename from elk/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml rename to ccs/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml diff --git a/elk/promptsource/templates/common_gen/templates.yaml b/ccs/promptsource/templates/common_gen/templates.yaml similarity index 100% rename from elk/promptsource/templates/common_gen/templates.yaml rename to ccs/promptsource/templates/common_gen/templates.yaml diff --git a/elk/promptsource/templates/commonsense_qa/templates.yaml b/ccs/promptsource/templates/commonsense_qa/templates.yaml similarity index 100% rename from elk/promptsource/templates/commonsense_qa/templates.yaml rename to ccs/promptsource/templates/commonsense_qa/templates.yaml diff --git a/elk/promptsource/templates/conv_ai/templates.yaml b/ccs/promptsource/templates/conv_ai/templates.yaml similarity index 100% rename from elk/promptsource/templates/conv_ai/templates.yaml rename to ccs/promptsource/templates/conv_ai/templates.yaml diff --git a/elk/promptsource/templates/conv_ai_2/templates.yaml b/ccs/promptsource/templates/conv_ai_2/templates.yaml similarity index 100% rename from elk/promptsource/templates/conv_ai_2/templates.yaml rename to ccs/promptsource/templates/conv_ai_2/templates.yaml diff --git a/elk/promptsource/templates/conv_ai_3/templates.yaml b/ccs/promptsource/templates/conv_ai_3/templates.yaml similarity index 100% rename from elk/promptsource/templates/conv_ai_3/templates.yaml rename to ccs/promptsource/templates/conv_ai_3/templates.yaml diff --git a/elk/promptsource/templates/coqa/templates.yaml b/ccs/promptsource/templates/coqa/templates.yaml similarity index 100% rename from elk/promptsource/templates/coqa/templates.yaml rename to ccs/promptsource/templates/coqa/templates.yaml diff --git a/elk/promptsource/templates/cord19/metadata/templates.yaml b/ccs/promptsource/templates/cord19/metadata/templates.yaml similarity index 100% rename from elk/promptsource/templates/cord19/metadata/templates.yaml rename to ccs/promptsource/templates/cord19/metadata/templates.yaml diff --git a/elk/promptsource/templates/cos_e/v1.0/templates.yaml b/ccs/promptsource/templates/cos_e/v1.0/templates.yaml similarity index 100% rename from elk/promptsource/templates/cos_e/v1.0/templates.yaml rename to ccs/promptsource/templates/cos_e/v1.0/templates.yaml diff --git a/elk/promptsource/templates/cos_e/v1.11/templates.yaml b/ccs/promptsource/templates/cos_e/v1.11/templates.yaml similarity index 100% rename from elk/promptsource/templates/cos_e/v1.11/templates.yaml rename to ccs/promptsource/templates/cos_e/v1.11/templates.yaml diff --git a/elk/promptsource/templates/cosmos_qa/templates.yaml b/ccs/promptsource/templates/cosmos_qa/templates.yaml similarity index 100% rename from elk/promptsource/templates/cosmos_qa/templates.yaml rename to ccs/promptsource/templates/cosmos_qa/templates.yaml diff --git a/elk/promptsource/templates/covid_qa_castorini/templates.yaml b/ccs/promptsource/templates/covid_qa_castorini/templates.yaml similarity index 100% rename from elk/promptsource/templates/covid_qa_castorini/templates.yaml rename to ccs/promptsource/templates/covid_qa_castorini/templates.yaml diff --git a/elk/promptsource/templates/craffel/openai_lambada/templates.yaml b/ccs/promptsource/templates/craffel/openai_lambada/templates.yaml similarity index 100% rename from elk/promptsource/templates/craffel/openai_lambada/templates.yaml rename to ccs/promptsource/templates/craffel/openai_lambada/templates.yaml diff --git a/elk/promptsource/templates/craigslist_bargains/templates.yaml b/ccs/promptsource/templates/craigslist_bargains/templates.yaml similarity index 100% rename from elk/promptsource/templates/craigslist_bargains/templates.yaml rename to ccs/promptsource/templates/craigslist_bargains/templates.yaml diff --git a/elk/promptsource/templates/crows_pairs/templates.yaml b/ccs/promptsource/templates/crows_pairs/templates.yaml similarity index 100% rename from elk/promptsource/templates/crows_pairs/templates.yaml rename to ccs/promptsource/templates/crows_pairs/templates.yaml diff --git a/elk/promptsource/templates/dbpedia_14/templates.yaml b/ccs/promptsource/templates/dbpedia_14/templates.yaml similarity index 100% rename from elk/promptsource/templates/dbpedia_14/templates.yaml rename to ccs/promptsource/templates/dbpedia_14/templates.yaml diff --git a/elk/promptsource/templates/discofuse/discofuse-sport/templates.yaml b/ccs/promptsource/templates/discofuse/discofuse-sport/templates.yaml similarity index 100% rename from elk/promptsource/templates/discofuse/discofuse-sport/templates.yaml rename to ccs/promptsource/templates/discofuse/discofuse-sport/templates.yaml diff --git a/elk/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml b/ccs/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml similarity index 100% rename from elk/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml rename to ccs/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml diff --git a/elk/promptsource/templates/discovery/discovery/templates.yaml b/ccs/promptsource/templates/discovery/discovery/templates.yaml similarity index 100% rename from elk/promptsource/templates/discovery/discovery/templates.yaml rename to ccs/promptsource/templates/discovery/discovery/templates.yaml diff --git a/elk/promptsource/templates/docred/templates.yaml b/ccs/promptsource/templates/docred/templates.yaml similarity index 100% rename from elk/promptsource/templates/docred/templates.yaml rename to ccs/promptsource/templates/docred/templates.yaml diff --git a/elk/promptsource/templates/dream/templates.yaml b/ccs/promptsource/templates/dream/templates.yaml similarity index 100% rename from elk/promptsource/templates/dream/templates.yaml rename to ccs/promptsource/templates/dream/templates.yaml diff --git a/elk/promptsource/templates/drop/templates.yaml b/ccs/promptsource/templates/drop/templates.yaml similarity index 100% rename from elk/promptsource/templates/drop/templates.yaml rename to ccs/promptsource/templates/drop/templates.yaml diff --git a/elk/promptsource/templates/duorc/ParaphraseRC/templates.yaml b/ccs/promptsource/templates/duorc/ParaphraseRC/templates.yaml similarity index 100% rename from elk/promptsource/templates/duorc/ParaphraseRC/templates.yaml rename to ccs/promptsource/templates/duorc/ParaphraseRC/templates.yaml diff --git a/elk/promptsource/templates/duorc/SelfRC/templates.yaml b/ccs/promptsource/templates/duorc/SelfRC/templates.yaml similarity index 100% rename from elk/promptsource/templates/duorc/SelfRC/templates.yaml rename to ccs/promptsource/templates/duorc/SelfRC/templates.yaml diff --git a/elk/promptsource/templates/e2e_nlg_cleaned/templates.yaml b/ccs/promptsource/templates/e2e_nlg_cleaned/templates.yaml similarity index 100% rename from elk/promptsource/templates/e2e_nlg_cleaned/templates.yaml rename to ccs/promptsource/templates/e2e_nlg_cleaned/templates.yaml diff --git a/elk/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml b/ccs/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml similarity index 100% rename from elk/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml rename to ccs/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml diff --git a/elk/promptsource/templates/emo/templates.yaml b/ccs/promptsource/templates/emo/templates.yaml similarity index 100% rename from elk/promptsource/templates/emo/templates.yaml rename to ccs/promptsource/templates/emo/templates.yaml diff --git a/elk/promptsource/templates/emotion/templates.yaml b/ccs/promptsource/templates/emotion/templates.yaml similarity index 100% rename from elk/promptsource/templates/emotion/templates.yaml rename to ccs/promptsource/templates/emotion/templates.yaml diff --git a/elk/promptsource/templates/enriched_web_nlg/en/templates.yaml b/ccs/promptsource/templates/enriched_web_nlg/en/templates.yaml similarity index 100% rename from elk/promptsource/templates/enriched_web_nlg/en/templates.yaml rename to ccs/promptsource/templates/enriched_web_nlg/en/templates.yaml diff --git a/elk/promptsource/templates/esnli/templates.yaml b/ccs/promptsource/templates/esnli/templates.yaml similarity index 100% rename from elk/promptsource/templates/esnli/templates.yaml rename to ccs/promptsource/templates/esnli/templates.yaml diff --git a/elk/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml b/ccs/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml similarity index 100% rename from elk/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml rename to ccs/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml diff --git a/elk/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml b/ccs/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml similarity index 100% rename from elk/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml rename to ccs/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml diff --git a/elk/promptsource/templates/fever/v1.0/templates.yaml b/ccs/promptsource/templates/fever/v1.0/templates.yaml similarity index 100% rename from elk/promptsource/templates/fever/v1.0/templates.yaml rename to ccs/promptsource/templates/fever/v1.0/templates.yaml diff --git a/elk/promptsource/templates/fever/v2.0/templates.yaml b/ccs/promptsource/templates/fever/v2.0/templates.yaml similarity index 100% rename from elk/promptsource/templates/fever/v2.0/templates.yaml rename to ccs/promptsource/templates/fever/v2.0/templates.yaml diff --git a/elk/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml b/ccs/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml similarity index 100% rename from elk/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml rename to ccs/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml diff --git a/elk/promptsource/templates/freebase_qa/templates.yaml b/ccs/promptsource/templates/freebase_qa/templates.yaml similarity index 100% rename from elk/promptsource/templates/freebase_qa/templates.yaml rename to ccs/promptsource/templates/freebase_qa/templates.yaml diff --git a/elk/promptsource/templates/generated_reviews_enth/templates.yaml b/ccs/promptsource/templates/generated_reviews_enth/templates.yaml similarity index 100% rename from elk/promptsource/templates/generated_reviews_enth/templates.yaml rename to ccs/promptsource/templates/generated_reviews_enth/templates.yaml diff --git a/elk/promptsource/templates/gigaword/templates.yaml b/ccs/promptsource/templates/gigaword/templates.yaml similarity index 100% rename from elk/promptsource/templates/gigaword/templates.yaml rename to ccs/promptsource/templates/gigaword/templates.yaml diff --git a/elk/promptsource/templates/glue/ax/templates.yaml b/ccs/promptsource/templates/glue/ax/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/ax/templates.yaml rename to ccs/promptsource/templates/glue/ax/templates.yaml diff --git a/elk/promptsource/templates/glue/cola/templates.yaml b/ccs/promptsource/templates/glue/cola/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/cola/templates.yaml rename to ccs/promptsource/templates/glue/cola/templates.yaml diff --git a/elk/promptsource/templates/glue/mnli/templates.yaml b/ccs/promptsource/templates/glue/mnli/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/mnli/templates.yaml rename to ccs/promptsource/templates/glue/mnli/templates.yaml diff --git a/elk/promptsource/templates/glue/mnli_matched/templates.yaml b/ccs/promptsource/templates/glue/mnli_matched/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/mnli_matched/templates.yaml rename to ccs/promptsource/templates/glue/mnli_matched/templates.yaml diff --git a/elk/promptsource/templates/glue/mnli_mismatched/templates.yaml b/ccs/promptsource/templates/glue/mnli_mismatched/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/mnli_mismatched/templates.yaml rename to ccs/promptsource/templates/glue/mnli_mismatched/templates.yaml diff --git a/elk/promptsource/templates/glue/mrpc/templates.yaml b/ccs/promptsource/templates/glue/mrpc/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/mrpc/templates.yaml rename to ccs/promptsource/templates/glue/mrpc/templates.yaml diff --git a/elk/promptsource/templates/glue/qnli/templates.yaml b/ccs/promptsource/templates/glue/qnli/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/qnli/templates.yaml rename to ccs/promptsource/templates/glue/qnli/templates.yaml diff --git a/elk/promptsource/templates/glue/qqp/templates.yaml b/ccs/promptsource/templates/glue/qqp/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/qqp/templates.yaml rename to ccs/promptsource/templates/glue/qqp/templates.yaml diff --git a/elk/promptsource/templates/glue/rte/templates.yaml b/ccs/promptsource/templates/glue/rte/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/rte/templates.yaml rename to ccs/promptsource/templates/glue/rte/templates.yaml diff --git a/elk/promptsource/templates/glue/sst2/templates.yaml b/ccs/promptsource/templates/glue/sst2/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/sst2/templates.yaml rename to ccs/promptsource/templates/glue/sst2/templates.yaml diff --git a/elk/promptsource/templates/glue/stsb/templates.yaml b/ccs/promptsource/templates/glue/stsb/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/stsb/templates.yaml rename to ccs/promptsource/templates/glue/stsb/templates.yaml diff --git a/elk/promptsource/templates/glue/wnli/templates.yaml b/ccs/promptsource/templates/glue/wnli/templates.yaml similarity index 100% rename from elk/promptsource/templates/glue/wnli/templates.yaml rename to ccs/promptsource/templates/glue/wnli/templates.yaml diff --git a/elk/promptsource/templates/google_wellformed_query/templates.yaml b/ccs/promptsource/templates/google_wellformed_query/templates.yaml similarity index 100% rename from elk/promptsource/templates/google_wellformed_query/templates.yaml rename to ccs/promptsource/templates/google_wellformed_query/templates.yaml diff --git a/elk/promptsource/templates/great_code/templates.yaml b/ccs/promptsource/templates/great_code/templates.yaml similarity index 100% rename from elk/promptsource/templates/great_code/templates.yaml rename to ccs/promptsource/templates/great_code/templates.yaml diff --git a/elk/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml b/ccs/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml similarity index 100% rename from elk/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml rename to ccs/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml diff --git a/elk/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml b/ccs/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml similarity index 100% rename from elk/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml rename to ccs/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml diff --git a/elk/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml b/ccs/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml similarity index 100% rename from elk/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml rename to ccs/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml diff --git a/elk/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml b/ccs/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml similarity index 100% rename from elk/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml rename to ccs/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml diff --git a/elk/promptsource/templates/gutenberg_time/templates.yaml b/ccs/promptsource/templates/gutenberg_time/templates.yaml similarity index 100% rename from elk/promptsource/templates/gutenberg_time/templates.yaml rename to ccs/promptsource/templates/gutenberg_time/templates.yaml diff --git a/elk/promptsource/templates/hans/templates.yaml b/ccs/promptsource/templates/hans/templates.yaml similarity index 100% rename from elk/promptsource/templates/hans/templates.yaml rename to ccs/promptsource/templates/hans/templates.yaml diff --git a/elk/promptsource/templates/hate_speech18/templates.yaml b/ccs/promptsource/templates/hate_speech18/templates.yaml similarity index 100% rename from elk/promptsource/templates/hate_speech18/templates.yaml rename to ccs/promptsource/templates/hate_speech18/templates.yaml diff --git a/elk/promptsource/templates/head_qa/en/templates.yaml b/ccs/promptsource/templates/head_qa/en/templates.yaml similarity index 100% rename from elk/promptsource/templates/head_qa/en/templates.yaml rename to ccs/promptsource/templates/head_qa/en/templates.yaml diff --git a/elk/promptsource/templates/health_fact/templates.yaml b/ccs/promptsource/templates/health_fact/templates.yaml similarity index 100% rename from elk/promptsource/templates/health_fact/templates.yaml rename to ccs/promptsource/templates/health_fact/templates.yaml diff --git a/elk/promptsource/templates/hellaswag/templates.yaml b/ccs/promptsource/templates/hellaswag/templates.yaml similarity index 100% rename from elk/promptsource/templates/hellaswag/templates.yaml rename to ccs/promptsource/templates/hellaswag/templates.yaml diff --git a/elk/promptsource/templates/hlgd/templates.yaml b/ccs/promptsource/templates/hlgd/templates.yaml similarity index 100% rename from elk/promptsource/templates/hlgd/templates.yaml rename to ccs/promptsource/templates/hlgd/templates.yaml diff --git a/elk/promptsource/templates/hotpot_qa/distractor/templates.yaml b/ccs/promptsource/templates/hotpot_qa/distractor/templates.yaml similarity index 100% rename from elk/promptsource/templates/hotpot_qa/distractor/templates.yaml rename to ccs/promptsource/templates/hotpot_qa/distractor/templates.yaml diff --git a/elk/promptsource/templates/hotpot_qa/fullwiki/templates.yaml b/ccs/promptsource/templates/hotpot_qa/fullwiki/templates.yaml similarity index 100% rename from elk/promptsource/templates/hotpot_qa/fullwiki/templates.yaml rename to ccs/promptsource/templates/hotpot_qa/fullwiki/templates.yaml diff --git a/elk/promptsource/templates/humicroedit/subtask-1/templates.yaml b/ccs/promptsource/templates/humicroedit/subtask-1/templates.yaml similarity index 100% rename from elk/promptsource/templates/humicroedit/subtask-1/templates.yaml rename to ccs/promptsource/templates/humicroedit/subtask-1/templates.yaml diff --git a/elk/promptsource/templates/humicroedit/subtask-2/templates.yaml b/ccs/promptsource/templates/humicroedit/subtask-2/templates.yaml similarity index 100% rename from elk/promptsource/templates/humicroedit/subtask-2/templates.yaml rename to ccs/promptsource/templates/humicroedit/subtask-2/templates.yaml diff --git a/elk/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml b/ccs/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml similarity index 100% rename from elk/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml rename to ccs/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml diff --git a/elk/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml b/ccs/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml similarity index 100% rename from elk/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml rename to ccs/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml diff --git a/elk/promptsource/templates/imdb/templates.yaml b/ccs/promptsource/templates/imdb/templates.yaml similarity index 100% rename from elk/promptsource/templates/imdb/templates.yaml rename to ccs/promptsource/templates/imdb/templates.yaml diff --git a/elk/promptsource/templates/jfleg/templates.yaml b/ccs/promptsource/templates/jfleg/templates.yaml similarity index 100% rename from elk/promptsource/templates/jfleg/templates.yaml rename to ccs/promptsource/templates/jfleg/templates.yaml diff --git a/elk/promptsource/templates/jigsaw_unintended_bias/templates.yaml b/ccs/promptsource/templates/jigsaw_unintended_bias/templates.yaml similarity index 100% rename from elk/promptsource/templates/jigsaw_unintended_bias/templates.yaml rename to ccs/promptsource/templates/jigsaw_unintended_bias/templates.yaml diff --git a/elk/promptsource/templates/kelm/templates.yaml b/ccs/promptsource/templates/kelm/templates.yaml similarity index 100% rename from elk/promptsource/templates/kelm/templates.yaml rename to ccs/promptsource/templates/kelm/templates.yaml diff --git a/elk/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml b/ccs/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml similarity index 100% rename from elk/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml rename to ccs/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml diff --git a/elk/promptsource/templates/kilt_tasks/nq/templates.yaml b/ccs/promptsource/templates/kilt_tasks/nq/templates.yaml similarity index 100% rename from elk/promptsource/templates/kilt_tasks/nq/templates.yaml rename to ccs/promptsource/templates/kilt_tasks/nq/templates.yaml diff --git a/elk/promptsource/templates/lama/trex/templates.yaml b/ccs/promptsource/templates/lama/trex/templates.yaml similarity index 100% rename from elk/promptsource/templates/lama/trex/templates.yaml rename to ccs/promptsource/templates/lama/trex/templates.yaml diff --git a/elk/promptsource/templates/lambada/templates.yaml b/ccs/promptsource/templates/lambada/templates.yaml similarity index 100% rename from elk/promptsource/templates/lambada/templates.yaml rename to ccs/promptsource/templates/lambada/templates.yaml diff --git a/elk/promptsource/templates/lauritowal/redefine_math/templates.yaml b/ccs/promptsource/templates/lauritowal/redefine_math/templates.yaml similarity index 100% rename from elk/promptsource/templates/lauritowal/redefine_math/templates.yaml rename to ccs/promptsource/templates/lauritowal/redefine_math/templates.yaml diff --git a/elk/promptsource/templates/liar/templates.yaml b/ccs/promptsource/templates/liar/templates.yaml similarity index 100% rename from elk/promptsource/templates/liar/templates.yaml rename to ccs/promptsource/templates/liar/templates.yaml diff --git a/elk/promptsource/templates/limit/templates.yaml b/ccs/promptsource/templates/limit/templates.yaml similarity index 100% rename from elk/promptsource/templates/limit/templates.yaml rename to ccs/promptsource/templates/limit/templates.yaml diff --git a/elk/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml b/ccs/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml similarity index 100% rename from elk/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml rename to ccs/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml diff --git a/elk/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml b/ccs/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml similarity index 100% rename from elk/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml rename to ccs/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml diff --git a/elk/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml b/ccs/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml similarity index 100% rename from elk/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml rename to ccs/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml diff --git a/elk/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml b/ccs/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml similarity index 100% rename from elk/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml rename to ccs/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml diff --git a/elk/promptsource/templates/math_qa/templates.yaml b/ccs/promptsource/templates/math_qa/templates.yaml similarity index 100% rename from elk/promptsource/templates/math_qa/templates.yaml rename to ccs/promptsource/templates/math_qa/templates.yaml diff --git a/elk/promptsource/templates/mc_taco/templates.yaml b/ccs/promptsource/templates/mc_taco/templates.yaml similarity index 100% rename from elk/promptsource/templates/mc_taco/templates.yaml rename to ccs/promptsource/templates/mc_taco/templates.yaml diff --git a/elk/promptsource/templates/mdd/task1_qa/templates.yaml b/ccs/promptsource/templates/mdd/task1_qa/templates.yaml similarity index 100% rename from elk/promptsource/templates/mdd/task1_qa/templates.yaml rename to ccs/promptsource/templates/mdd/task1_qa/templates.yaml diff --git a/elk/promptsource/templates/mdd/task2_recs/templates.yaml b/ccs/promptsource/templates/mdd/task2_recs/templates.yaml similarity index 100% rename from elk/promptsource/templates/mdd/task2_recs/templates.yaml rename to ccs/promptsource/templates/mdd/task2_recs/templates.yaml diff --git a/elk/promptsource/templates/mdd/task3_qarecs/templates.yaml b/ccs/promptsource/templates/mdd/task3_qarecs/templates.yaml similarity index 100% rename from elk/promptsource/templates/mdd/task3_qarecs/templates.yaml rename to ccs/promptsource/templates/mdd/task3_qarecs/templates.yaml diff --git a/elk/promptsource/templates/medal/templates.yaml b/ccs/promptsource/templates/medal/templates.yaml similarity index 100% rename from elk/promptsource/templates/medal/templates.yaml rename to ccs/promptsource/templates/medal/templates.yaml diff --git a/elk/promptsource/templates/medical_questions_pairs/templates.yaml b/ccs/promptsource/templates/medical_questions_pairs/templates.yaml similarity index 100% rename from elk/promptsource/templates/medical_questions_pairs/templates.yaml rename to ccs/promptsource/templates/medical_questions_pairs/templates.yaml diff --git a/elk/promptsource/templates/meta_woz/dialogues/templates.yaml b/ccs/promptsource/templates/meta_woz/dialogues/templates.yaml similarity index 100% rename from elk/promptsource/templates/meta_woz/dialogues/templates.yaml rename to ccs/promptsource/templates/meta_woz/dialogues/templates.yaml diff --git a/elk/promptsource/templates/mocha/templates.yaml b/ccs/promptsource/templates/mocha/templates.yaml similarity index 100% rename from elk/promptsource/templates/mocha/templates.yaml rename to ccs/promptsource/templates/mocha/templates.yaml diff --git a/elk/promptsource/templates/movie_rationales/templates.yaml b/ccs/promptsource/templates/movie_rationales/templates.yaml similarity index 100% rename from elk/promptsource/templates/movie_rationales/templates.yaml rename to ccs/promptsource/templates/movie_rationales/templates.yaml diff --git a/elk/promptsource/templates/multi_news/templates.yaml b/ccs/promptsource/templates/multi_news/templates.yaml similarity index 100% rename from elk/promptsource/templates/multi_news/templates.yaml rename to ccs/promptsource/templates/multi_news/templates.yaml diff --git a/elk/promptsource/templates/multi_nli/templates.yaml b/ccs/promptsource/templates/multi_nli/templates.yaml similarity index 100% rename from elk/promptsource/templates/multi_nli/templates.yaml rename to ccs/promptsource/templates/multi_nli/templates.yaml diff --git a/elk/promptsource/templates/multi_x_science_sum/templates.yaml b/ccs/promptsource/templates/multi_x_science_sum/templates.yaml similarity index 100% rename from elk/promptsource/templates/multi_x_science_sum/templates.yaml rename to ccs/promptsource/templates/multi_x_science_sum/templates.yaml diff --git a/elk/promptsource/templates/mwsc/templates.yaml b/ccs/promptsource/templates/mwsc/templates.yaml similarity index 100% rename from elk/promptsource/templates/mwsc/templates.yaml rename to ccs/promptsource/templates/mwsc/templates.yaml diff --git a/elk/promptsource/templates/narrativeqa/templates.yaml b/ccs/promptsource/templates/narrativeqa/templates.yaml similarity index 100% rename from elk/promptsource/templates/narrativeqa/templates.yaml rename to ccs/promptsource/templates/narrativeqa/templates.yaml diff --git a/elk/promptsource/templates/ncbi_disease/templates.yaml b/ccs/promptsource/templates/ncbi_disease/templates.yaml similarity index 100% rename from elk/promptsource/templates/ncbi_disease/templates.yaml rename to ccs/promptsource/templates/ncbi_disease/templates.yaml diff --git a/elk/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml b/ccs/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml similarity index 100% rename from elk/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml rename to ccs/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml diff --git a/elk/promptsource/templates/newspop/templates.yaml b/ccs/promptsource/templates/newspop/templates.yaml similarity index 100% rename from elk/promptsource/templates/newspop/templates.yaml rename to ccs/promptsource/templates/newspop/templates.yaml diff --git a/elk/promptsource/templates/nlu_evaluation_data/templates.yaml b/ccs/promptsource/templates/nlu_evaluation_data/templates.yaml similarity index 100% rename from elk/promptsource/templates/nlu_evaluation_data/templates.yaml rename to ccs/promptsource/templates/nlu_evaluation_data/templates.yaml diff --git a/elk/promptsource/templates/nq_open/templates.yaml b/ccs/promptsource/templates/nq_open/templates.yaml similarity index 100% rename from elk/promptsource/templates/nq_open/templates.yaml rename to ccs/promptsource/templates/nq_open/templates.yaml diff --git a/elk/promptsource/templates/numer_sense/templates.yaml b/ccs/promptsource/templates/numer_sense/templates.yaml similarity index 100% rename from elk/promptsource/templates/numer_sense/templates.yaml rename to ccs/promptsource/templates/numer_sense/templates.yaml diff --git a/elk/promptsource/templates/onestop_english/templates.yaml b/ccs/promptsource/templates/onestop_english/templates.yaml similarity index 100% rename from elk/promptsource/templates/onestop_english/templates.yaml rename to ccs/promptsource/templates/onestop_english/templates.yaml diff --git a/elk/promptsource/templates/openai_humaneval/templates.yaml b/ccs/promptsource/templates/openai_humaneval/templates.yaml similarity index 100% rename from elk/promptsource/templates/openai_humaneval/templates.yaml rename to ccs/promptsource/templates/openai_humaneval/templates.yaml diff --git a/elk/promptsource/templates/openbookqa/additional/templates.yaml b/ccs/promptsource/templates/openbookqa/additional/templates.yaml similarity index 100% rename from elk/promptsource/templates/openbookqa/additional/templates.yaml rename to ccs/promptsource/templates/openbookqa/additional/templates.yaml diff --git a/elk/promptsource/templates/openbookqa/main/templates.yaml b/ccs/promptsource/templates/openbookqa/main/templates.yaml similarity index 100% rename from elk/promptsource/templates/openbookqa/main/templates.yaml rename to ccs/promptsource/templates/openbookqa/main/templates.yaml diff --git a/elk/promptsource/templates/paws-x/en/templates.yaml b/ccs/promptsource/templates/paws-x/en/templates.yaml similarity index 100% rename from elk/promptsource/templates/paws-x/en/templates.yaml rename to ccs/promptsource/templates/paws-x/en/templates.yaml diff --git a/elk/promptsource/templates/paws/labeled_final/templates.yaml b/ccs/promptsource/templates/paws/labeled_final/templates.yaml similarity index 100% rename from elk/promptsource/templates/paws/labeled_final/templates.yaml rename to ccs/promptsource/templates/paws/labeled_final/templates.yaml diff --git a/elk/promptsource/templates/paws/labeled_swap/templates.yaml b/ccs/promptsource/templates/paws/labeled_swap/templates.yaml similarity index 100% rename from elk/promptsource/templates/paws/labeled_swap/templates.yaml rename to ccs/promptsource/templates/paws/labeled_swap/templates.yaml diff --git a/elk/promptsource/templates/paws/unlabeled_final/templates.yaml b/ccs/promptsource/templates/paws/unlabeled_final/templates.yaml similarity index 100% rename from elk/promptsource/templates/paws/unlabeled_final/templates.yaml rename to ccs/promptsource/templates/paws/unlabeled_final/templates.yaml diff --git a/elk/promptsource/templates/piqa/templates.yaml b/ccs/promptsource/templates/piqa/templates.yaml similarity index 100% rename from elk/promptsource/templates/piqa/templates.yaml rename to ccs/promptsource/templates/piqa/templates.yaml diff --git a/elk/promptsource/templates/poem_sentiment/templates.yaml b/ccs/promptsource/templates/poem_sentiment/templates.yaml similarity index 100% rename from elk/promptsource/templates/poem_sentiment/templates.yaml rename to ccs/promptsource/templates/poem_sentiment/templates.yaml diff --git a/elk/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml b/ccs/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml similarity index 100% rename from elk/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml rename to ccs/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml diff --git a/elk/promptsource/templates/qa_srl/templates.yaml b/ccs/promptsource/templates/qa_srl/templates.yaml similarity index 100% rename from elk/promptsource/templates/qa_srl/templates.yaml rename to ccs/promptsource/templates/qa_srl/templates.yaml diff --git a/elk/promptsource/templates/qa_zre/templates.yaml b/ccs/promptsource/templates/qa_zre/templates.yaml similarity index 100% rename from elk/promptsource/templates/qa_zre/templates.yaml rename to ccs/promptsource/templates/qa_zre/templates.yaml diff --git a/elk/promptsource/templates/qasc/templates.yaml b/ccs/promptsource/templates/qasc/templates.yaml similarity index 100% rename from elk/promptsource/templates/qasc/templates.yaml rename to ccs/promptsource/templates/qasc/templates.yaml diff --git a/elk/promptsource/templates/qed/templates.yaml b/ccs/promptsource/templates/qed/templates.yaml similarity index 100% rename from elk/promptsource/templates/qed/templates.yaml rename to ccs/promptsource/templates/qed/templates.yaml diff --git a/elk/promptsource/templates/quac/templates.yaml b/ccs/promptsource/templates/quac/templates.yaml similarity index 100% rename from elk/promptsource/templates/quac/templates.yaml rename to ccs/promptsource/templates/quac/templates.yaml diff --git a/elk/promptsource/templates/quail/templates.yaml b/ccs/promptsource/templates/quail/templates.yaml similarity index 100% rename from elk/promptsource/templates/quail/templates.yaml rename to ccs/promptsource/templates/quail/templates.yaml diff --git a/elk/promptsource/templates/quarel/templates.yaml b/ccs/promptsource/templates/quarel/templates.yaml similarity index 100% rename from elk/promptsource/templates/quarel/templates.yaml rename to ccs/promptsource/templates/quarel/templates.yaml diff --git a/elk/promptsource/templates/quartz/templates.yaml b/ccs/promptsource/templates/quartz/templates.yaml similarity index 100% rename from elk/promptsource/templates/quartz/templates.yaml rename to ccs/promptsource/templates/quartz/templates.yaml diff --git a/elk/promptsource/templates/quora/templates.yaml b/ccs/promptsource/templates/quora/templates.yaml similarity index 100% rename from elk/promptsource/templates/quora/templates.yaml rename to ccs/promptsource/templates/quora/templates.yaml diff --git a/elk/promptsource/templates/quoref/templates.yaml b/ccs/promptsource/templates/quoref/templates.yaml similarity index 100% rename from elk/promptsource/templates/quoref/templates.yaml rename to ccs/promptsource/templates/quoref/templates.yaml diff --git a/elk/promptsource/templates/race/all/templates.yaml b/ccs/promptsource/templates/race/all/templates.yaml similarity index 100% rename from elk/promptsource/templates/race/all/templates.yaml rename to ccs/promptsource/templates/race/all/templates.yaml diff --git a/elk/promptsource/templates/race/high/templates.yaml b/ccs/promptsource/templates/race/high/templates.yaml similarity index 100% rename from elk/promptsource/templates/race/high/templates.yaml rename to ccs/promptsource/templates/race/high/templates.yaml diff --git a/elk/promptsource/templates/race/middle/templates.yaml b/ccs/promptsource/templates/race/middle/templates.yaml similarity index 100% rename from elk/promptsource/templates/race/middle/templates.yaml rename to ccs/promptsource/templates/race/middle/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/boolq_ar/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_ar/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/boolq_ar/templates.yaml rename to ccs/promptsource/templates/reaganjlee/boolq_ar/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/boolq_es/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_es/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/boolq_es/templates.yaml rename to ccs/promptsource/templates/reaganjlee/boolq_es/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/boolq_fr/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_fr/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/boolq_fr/templates.yaml rename to ccs/promptsource/templates/reaganjlee/boolq_fr/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/boolq_pt/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_pt/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/boolq_pt/templates.yaml rename to ccs/promptsource/templates/reaganjlee/boolq_pt/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/boolq_zh/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_zh/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/boolq_zh/templates.yaml rename to ccs/promptsource/templates/reaganjlee/boolq_zh/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml similarity index 100% rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml diff --git a/elk/promptsource/templates/riddle_sense/templates.yaml b/ccs/promptsource/templates/riddle_sense/templates.yaml similarity index 100% rename from elk/promptsource/templates/riddle_sense/templates.yaml rename to ccs/promptsource/templates/riddle_sense/templates.yaml diff --git a/elk/promptsource/templates/ropes/templates.yaml b/ccs/promptsource/templates/ropes/templates.yaml similarity index 100% rename from elk/promptsource/templates/ropes/templates.yaml rename to ccs/promptsource/templates/ropes/templates.yaml diff --git a/elk/promptsource/templates/rotten_tomatoes/templates.yaml b/ccs/promptsource/templates/rotten_tomatoes/templates.yaml similarity index 100% rename from elk/promptsource/templates/rotten_tomatoes/templates.yaml rename to ccs/promptsource/templates/rotten_tomatoes/templates.yaml diff --git a/elk/promptsource/templates/samsum/templates.yaml b/ccs/promptsource/templates/samsum/templates.yaml similarity index 100% rename from elk/promptsource/templates/samsum/templates.yaml rename to ccs/promptsource/templates/samsum/templates.yaml diff --git a/elk/promptsource/templates/scan/addprim_jump/templates.yaml b/ccs/promptsource/templates/scan/addprim_jump/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/addprim_jump/templates.yaml rename to ccs/promptsource/templates/scan/addprim_jump/templates.yaml diff --git a/elk/promptsource/templates/scan/addprim_turn_left/templates.yaml b/ccs/promptsource/templates/scan/addprim_turn_left/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/addprim_turn_left/templates.yaml rename to ccs/promptsource/templates/scan/addprim_turn_left/templates.yaml diff --git a/elk/promptsource/templates/scan/filler_num0/templates.yaml b/ccs/promptsource/templates/scan/filler_num0/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/filler_num0/templates.yaml rename to ccs/promptsource/templates/scan/filler_num0/templates.yaml diff --git a/elk/promptsource/templates/scan/filler_num1/templates.yaml b/ccs/promptsource/templates/scan/filler_num1/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/filler_num1/templates.yaml rename to ccs/promptsource/templates/scan/filler_num1/templates.yaml diff --git a/elk/promptsource/templates/scan/filler_num2/templates.yaml b/ccs/promptsource/templates/scan/filler_num2/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/filler_num2/templates.yaml rename to ccs/promptsource/templates/scan/filler_num2/templates.yaml diff --git a/elk/promptsource/templates/scan/filler_num3/templates.yaml b/ccs/promptsource/templates/scan/filler_num3/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/filler_num3/templates.yaml rename to ccs/promptsource/templates/scan/filler_num3/templates.yaml diff --git a/elk/promptsource/templates/scan/length/templates.yaml b/ccs/promptsource/templates/scan/length/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/length/templates.yaml rename to ccs/promptsource/templates/scan/length/templates.yaml diff --git a/elk/promptsource/templates/scan/simple/templates.yaml b/ccs/promptsource/templates/scan/simple/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/simple/templates.yaml rename to ccs/promptsource/templates/scan/simple/templates.yaml diff --git a/elk/promptsource/templates/scan/template_around_right/templates.yaml b/ccs/promptsource/templates/scan/template_around_right/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/template_around_right/templates.yaml rename to ccs/promptsource/templates/scan/template_around_right/templates.yaml diff --git a/elk/promptsource/templates/scan/template_jump_around_right/templates.yaml b/ccs/promptsource/templates/scan/template_jump_around_right/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/template_jump_around_right/templates.yaml rename to ccs/promptsource/templates/scan/template_jump_around_right/templates.yaml diff --git a/elk/promptsource/templates/scan/template_opposite_right/templates.yaml b/ccs/promptsource/templates/scan/template_opposite_right/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/template_opposite_right/templates.yaml rename to ccs/promptsource/templates/scan/template_opposite_right/templates.yaml diff --git a/elk/promptsource/templates/scan/template_right/templates.yaml b/ccs/promptsource/templates/scan/template_right/templates.yaml similarity index 100% rename from elk/promptsource/templates/scan/template_right/templates.yaml rename to ccs/promptsource/templates/scan/template_right/templates.yaml diff --git a/elk/promptsource/templates/scicite/templates.yaml b/ccs/promptsource/templates/scicite/templates.yaml similarity index 100% rename from elk/promptsource/templates/scicite/templates.yaml rename to ccs/promptsource/templates/scicite/templates.yaml diff --git a/elk/promptsource/templates/scientific_papers/arxiv/templates.yaml b/ccs/promptsource/templates/scientific_papers/arxiv/templates.yaml similarity index 100% rename from elk/promptsource/templates/scientific_papers/arxiv/templates.yaml rename to ccs/promptsource/templates/scientific_papers/arxiv/templates.yaml diff --git a/elk/promptsource/templates/scientific_papers/pubmed/templates.yaml b/ccs/promptsource/templates/scientific_papers/pubmed/templates.yaml similarity index 100% rename from elk/promptsource/templates/scientific_papers/pubmed/templates.yaml rename to ccs/promptsource/templates/scientific_papers/pubmed/templates.yaml diff --git a/elk/promptsource/templates/sciq/templates.yaml b/ccs/promptsource/templates/sciq/templates.yaml similarity index 100% rename from elk/promptsource/templates/sciq/templates.yaml rename to ccs/promptsource/templates/sciq/templates.yaml diff --git a/elk/promptsource/templates/scitail/snli_format/templates.yaml b/ccs/promptsource/templates/scitail/snli_format/templates.yaml similarity index 100% rename from elk/promptsource/templates/scitail/snli_format/templates.yaml rename to ccs/promptsource/templates/scitail/snli_format/templates.yaml diff --git a/elk/promptsource/templates/scitail/tsv_format/templates.yaml b/ccs/promptsource/templates/scitail/tsv_format/templates.yaml similarity index 100% rename from elk/promptsource/templates/scitail/tsv_format/templates.yaml rename to ccs/promptsource/templates/scitail/tsv_format/templates.yaml diff --git a/elk/promptsource/templates/scitldr/Abstract/templates.yaml b/ccs/promptsource/templates/scitldr/Abstract/templates.yaml similarity index 100% rename from elk/promptsource/templates/scitldr/Abstract/templates.yaml rename to ccs/promptsource/templates/scitldr/Abstract/templates.yaml diff --git a/elk/promptsource/templates/selqa/answer_selection_analysis/templates.yaml b/ccs/promptsource/templates/selqa/answer_selection_analysis/templates.yaml similarity index 100% rename from elk/promptsource/templates/selqa/answer_selection_analysis/templates.yaml rename to ccs/promptsource/templates/selqa/answer_selection_analysis/templates.yaml diff --git a/elk/promptsource/templates/sem_eval_2010_task_8/templates.yaml b/ccs/promptsource/templates/sem_eval_2010_task_8/templates.yaml similarity index 100% rename from elk/promptsource/templates/sem_eval_2010_task_8/templates.yaml rename to ccs/promptsource/templates/sem_eval_2010_task_8/templates.yaml diff --git a/elk/promptsource/templates/sem_eval_2014_task_1/templates.yaml b/ccs/promptsource/templates/sem_eval_2014_task_1/templates.yaml similarity index 100% rename from elk/promptsource/templates/sem_eval_2014_task_1/templates.yaml rename to ccs/promptsource/templates/sem_eval_2014_task_1/templates.yaml diff --git a/elk/promptsource/templates/sent_comp/templates.yaml b/ccs/promptsource/templates/sent_comp/templates.yaml similarity index 100% rename from elk/promptsource/templates/sent_comp/templates.yaml rename to ccs/promptsource/templates/sent_comp/templates.yaml diff --git a/elk/promptsource/templates/sick/templates.yaml b/ccs/promptsource/templates/sick/templates.yaml similarity index 100% rename from elk/promptsource/templates/sick/templates.yaml rename to ccs/promptsource/templates/sick/templates.yaml diff --git a/elk/promptsource/templates/sms_spam/templates.yaml b/ccs/promptsource/templates/sms_spam/templates.yaml similarity index 100% rename from elk/promptsource/templates/sms_spam/templates.yaml rename to ccs/promptsource/templates/sms_spam/templates.yaml diff --git a/elk/promptsource/templates/snips_built_in_intents/templates.yaml b/ccs/promptsource/templates/snips_built_in_intents/templates.yaml similarity index 100% rename from elk/promptsource/templates/snips_built_in_intents/templates.yaml rename to ccs/promptsource/templates/snips_built_in_intents/templates.yaml diff --git a/elk/promptsource/templates/snli/templates.yaml b/ccs/promptsource/templates/snli/templates.yaml similarity index 100% rename from elk/promptsource/templates/snli/templates.yaml rename to ccs/promptsource/templates/snli/templates.yaml diff --git a/elk/promptsource/templates/social_i_qa/templates.yaml b/ccs/promptsource/templates/social_i_qa/templates.yaml similarity index 100% rename from elk/promptsource/templates/social_i_qa/templates.yaml rename to ccs/promptsource/templates/social_i_qa/templates.yaml diff --git a/elk/promptsource/templates/species_800/templates.yaml b/ccs/promptsource/templates/species_800/templates.yaml similarity index 100% rename from elk/promptsource/templates/species_800/templates.yaml rename to ccs/promptsource/templates/species_800/templates.yaml diff --git a/elk/promptsource/templates/squad/templates.yaml b/ccs/promptsource/templates/squad/templates.yaml similarity index 100% rename from elk/promptsource/templates/squad/templates.yaml rename to ccs/promptsource/templates/squad/templates.yaml diff --git a/elk/promptsource/templates/squad_adversarial/AddSent/templates.yaml b/ccs/promptsource/templates/squad_adversarial/AddSent/templates.yaml similarity index 100% rename from elk/promptsource/templates/squad_adversarial/AddSent/templates.yaml rename to ccs/promptsource/templates/squad_adversarial/AddSent/templates.yaml diff --git a/elk/promptsource/templates/squad_v2/templates.yaml b/ccs/promptsource/templates/squad_v2/templates.yaml similarity index 100% rename from elk/promptsource/templates/squad_v2/templates.yaml rename to ccs/promptsource/templates/squad_v2/templates.yaml diff --git a/elk/promptsource/templates/squadshifts/amazon/templates.yaml b/ccs/promptsource/templates/squadshifts/amazon/templates.yaml similarity index 100% rename from elk/promptsource/templates/squadshifts/amazon/templates.yaml rename to ccs/promptsource/templates/squadshifts/amazon/templates.yaml diff --git a/elk/promptsource/templates/squadshifts/new_wiki/templates.yaml b/ccs/promptsource/templates/squadshifts/new_wiki/templates.yaml similarity index 100% rename from elk/promptsource/templates/squadshifts/new_wiki/templates.yaml rename to ccs/promptsource/templates/squadshifts/new_wiki/templates.yaml diff --git a/elk/promptsource/templates/squadshifts/nyt/templates.yaml b/ccs/promptsource/templates/squadshifts/nyt/templates.yaml similarity index 100% rename from elk/promptsource/templates/squadshifts/nyt/templates.yaml rename to ccs/promptsource/templates/squadshifts/nyt/templates.yaml diff --git a/elk/promptsource/templates/sst/default/templates.yaml b/ccs/promptsource/templates/sst/default/templates.yaml similarity index 100% rename from elk/promptsource/templates/sst/default/templates.yaml rename to ccs/promptsource/templates/sst/default/templates.yaml diff --git a/elk/promptsource/templates/sst2/templates.yaml b/ccs/promptsource/templates/sst2/templates.yaml similarity index 100% rename from elk/promptsource/templates/sst2/templates.yaml rename to ccs/promptsource/templates/sst2/templates.yaml diff --git a/elk/promptsource/templates/story_cloze/2016/templates.yaml b/ccs/promptsource/templates/story_cloze/2016/templates.yaml similarity index 100% rename from elk/promptsource/templates/story_cloze/2016/templates.yaml rename to ccs/promptsource/templates/story_cloze/2016/templates.yaml diff --git a/elk/promptsource/templates/stsb_multi_mt/en/templates.yaml b/ccs/promptsource/templates/stsb_multi_mt/en/templates.yaml similarity index 100% rename from elk/promptsource/templates/stsb_multi_mt/en/templates.yaml rename to ccs/promptsource/templates/stsb_multi_mt/en/templates.yaml diff --git a/elk/promptsource/templates/subjqa/books/templates.yaml b/ccs/promptsource/templates/subjqa/books/templates.yaml similarity index 100% rename from elk/promptsource/templates/subjqa/books/templates.yaml rename to ccs/promptsource/templates/subjqa/books/templates.yaml diff --git a/elk/promptsource/templates/subjqa/electronics/templates.yaml b/ccs/promptsource/templates/subjqa/electronics/templates.yaml similarity index 100% rename from elk/promptsource/templates/subjqa/electronics/templates.yaml rename to ccs/promptsource/templates/subjqa/electronics/templates.yaml diff --git a/elk/promptsource/templates/subjqa/grocery/templates.yaml b/ccs/promptsource/templates/subjqa/grocery/templates.yaml similarity index 100% rename from elk/promptsource/templates/subjqa/grocery/templates.yaml rename to ccs/promptsource/templates/subjqa/grocery/templates.yaml diff --git a/elk/promptsource/templates/subjqa/movies/templates.yaml b/ccs/promptsource/templates/subjqa/movies/templates.yaml similarity index 100% rename from elk/promptsource/templates/subjqa/movies/templates.yaml rename to ccs/promptsource/templates/subjqa/movies/templates.yaml diff --git a/elk/promptsource/templates/subjqa/restaurants/templates.yaml b/ccs/promptsource/templates/subjqa/restaurants/templates.yaml similarity index 100% rename from elk/promptsource/templates/subjqa/restaurants/templates.yaml rename to ccs/promptsource/templates/subjqa/restaurants/templates.yaml diff --git a/elk/promptsource/templates/subjqa/tripadvisor/templates.yaml b/ccs/promptsource/templates/subjqa/tripadvisor/templates.yaml similarity index 100% rename from elk/promptsource/templates/subjqa/tripadvisor/templates.yaml rename to ccs/promptsource/templates/subjqa/tripadvisor/templates.yaml diff --git a/elk/promptsource/templates/super_glue/axb/templates.yaml b/ccs/promptsource/templates/super_glue/axb/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/axb/templates.yaml rename to ccs/promptsource/templates/super_glue/axb/templates.yaml diff --git a/elk/promptsource/templates/super_glue/axg/templates.yaml b/ccs/promptsource/templates/super_glue/axg/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/axg/templates.yaml rename to ccs/promptsource/templates/super_glue/axg/templates.yaml diff --git a/elk/promptsource/templates/super_glue/boolq/templates.yaml b/ccs/promptsource/templates/super_glue/boolq/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/boolq/templates.yaml rename to ccs/promptsource/templates/super_glue/boolq/templates.yaml diff --git a/elk/promptsource/templates/super_glue/cb/templates.yaml b/ccs/promptsource/templates/super_glue/cb/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/cb/templates.yaml rename to ccs/promptsource/templates/super_glue/cb/templates.yaml diff --git a/elk/promptsource/templates/super_glue/copa/templates.yaml b/ccs/promptsource/templates/super_glue/copa/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/copa/templates.yaml rename to ccs/promptsource/templates/super_glue/copa/templates.yaml diff --git a/elk/promptsource/templates/super_glue/multirc/templates.yaml b/ccs/promptsource/templates/super_glue/multirc/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/multirc/templates.yaml rename to ccs/promptsource/templates/super_glue/multirc/templates.yaml diff --git a/elk/promptsource/templates/super_glue/record/templates.yaml b/ccs/promptsource/templates/super_glue/record/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/record/templates.yaml rename to ccs/promptsource/templates/super_glue/record/templates.yaml diff --git a/elk/promptsource/templates/super_glue/rte/templates.yaml b/ccs/promptsource/templates/super_glue/rte/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/rte/templates.yaml rename to ccs/promptsource/templates/super_glue/rte/templates.yaml diff --git a/elk/promptsource/templates/super_glue/wic/templates.yaml b/ccs/promptsource/templates/super_glue/wic/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/wic/templates.yaml rename to ccs/promptsource/templates/super_glue/wic/templates.yaml diff --git a/elk/promptsource/templates/super_glue/wsc.fixed/templates.yaml b/ccs/promptsource/templates/super_glue/wsc.fixed/templates.yaml similarity index 100% rename from elk/promptsource/templates/super_glue/wsc.fixed/templates.yaml rename to ccs/promptsource/templates/super_glue/wsc.fixed/templates.yaml diff --git a/elk/promptsource/templates/swag/regular/templates.yaml b/ccs/promptsource/templates/swag/regular/templates.yaml similarity index 100% rename from elk/promptsource/templates/swag/regular/templates.yaml rename to ccs/promptsource/templates/swag/regular/templates.yaml diff --git a/elk/promptsource/templates/tab_fact/tab_fact/templates.yaml b/ccs/promptsource/templates/tab_fact/tab_fact/templates.yaml similarity index 100% rename from elk/promptsource/templates/tab_fact/tab_fact/templates.yaml rename to ccs/promptsource/templates/tab_fact/tab_fact/templates.yaml diff --git a/elk/promptsource/templates/tmu_gfm_dataset/templates.yaml b/ccs/promptsource/templates/tmu_gfm_dataset/templates.yaml similarity index 100% rename from elk/promptsource/templates/tmu_gfm_dataset/templates.yaml rename to ccs/promptsource/templates/tmu_gfm_dataset/templates.yaml diff --git a/elk/promptsource/templates/trec/templates.yaml b/ccs/promptsource/templates/trec/templates.yaml similarity index 100% rename from elk/promptsource/templates/trec/templates.yaml rename to ccs/promptsource/templates/trec/templates.yaml diff --git a/elk/promptsource/templates/trivia_qa/unfiltered/templates.yaml b/ccs/promptsource/templates/trivia_qa/unfiltered/templates.yaml similarity index 100% rename from elk/promptsource/templates/trivia_qa/unfiltered/templates.yaml rename to ccs/promptsource/templates/trivia_qa/unfiltered/templates.yaml diff --git a/elk/promptsource/templates/turk/templates.yaml b/ccs/promptsource/templates/turk/templates.yaml similarity index 100% rename from elk/promptsource/templates/turk/templates.yaml rename to ccs/promptsource/templates/turk/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/emoji/templates.yaml b/ccs/promptsource/templates/tweet_eval/emoji/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/emoji/templates.yaml rename to ccs/promptsource/templates/tweet_eval/emoji/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/emotion/templates.yaml b/ccs/promptsource/templates/tweet_eval/emotion/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/emotion/templates.yaml rename to ccs/promptsource/templates/tweet_eval/emotion/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/hate/templates.yaml b/ccs/promptsource/templates/tweet_eval/hate/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/hate/templates.yaml rename to ccs/promptsource/templates/tweet_eval/hate/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/irony/templates.yaml b/ccs/promptsource/templates/tweet_eval/irony/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/irony/templates.yaml rename to ccs/promptsource/templates/tweet_eval/irony/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/offensive/templates.yaml b/ccs/promptsource/templates/tweet_eval/offensive/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/offensive/templates.yaml rename to ccs/promptsource/templates/tweet_eval/offensive/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/sentiment/templates.yaml b/ccs/promptsource/templates/tweet_eval/sentiment/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/sentiment/templates.yaml rename to ccs/promptsource/templates/tweet_eval/sentiment/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/stance_abortion/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_abortion/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/stance_abortion/templates.yaml rename to ccs/promptsource/templates/tweet_eval/stance_abortion/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/stance_atheism/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_atheism/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/stance_atheism/templates.yaml rename to ccs/promptsource/templates/tweet_eval/stance_atheism/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/stance_climate/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_climate/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/stance_climate/templates.yaml rename to ccs/promptsource/templates/tweet_eval/stance_climate/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/stance_feminist/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_feminist/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/stance_feminist/templates.yaml rename to ccs/promptsource/templates/tweet_eval/stance_feminist/templates.yaml diff --git a/elk/promptsource/templates/tweet_eval/stance_hillary/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_hillary/templates.yaml similarity index 100% rename from elk/promptsource/templates/tweet_eval/stance_hillary/templates.yaml rename to ccs/promptsource/templates/tweet_eval/stance_hillary/templates.yaml diff --git a/elk/promptsource/templates/tydiqa/primary_task/templates.yaml b/ccs/promptsource/templates/tydiqa/primary_task/templates.yaml similarity index 100% rename from elk/promptsource/templates/tydiqa/primary_task/templates.yaml rename to ccs/promptsource/templates/tydiqa/primary_task/templates.yaml diff --git a/elk/promptsource/templates/tydiqa/secondary_task/templates.yaml b/ccs/promptsource/templates/tydiqa/secondary_task/templates.yaml similarity index 100% rename from elk/promptsource/templates/tydiqa/secondary_task/templates.yaml rename to ccs/promptsource/templates/tydiqa/secondary_task/templates.yaml diff --git a/elk/promptsource/templates/web_questions/templates.yaml b/ccs/promptsource/templates/web_questions/templates.yaml similarity index 100% rename from elk/promptsource/templates/web_questions/templates.yaml rename to ccs/promptsource/templates/web_questions/templates.yaml diff --git a/elk/promptsource/templates/wiki_bio/templates.yaml b/ccs/promptsource/templates/wiki_bio/templates.yaml similarity index 100% rename from elk/promptsource/templates/wiki_bio/templates.yaml rename to ccs/promptsource/templates/wiki_bio/templates.yaml diff --git a/elk/promptsource/templates/wiki_hop/masked/templates.yaml b/ccs/promptsource/templates/wiki_hop/masked/templates.yaml similarity index 100% rename from elk/promptsource/templates/wiki_hop/masked/templates.yaml rename to ccs/promptsource/templates/wiki_hop/masked/templates.yaml diff --git a/elk/promptsource/templates/wiki_hop/original/templates.yaml b/ccs/promptsource/templates/wiki_hop/original/templates.yaml similarity index 100% rename from elk/promptsource/templates/wiki_hop/original/templates.yaml rename to ccs/promptsource/templates/wiki_hop/original/templates.yaml diff --git a/elk/promptsource/templates/wiki_qa/templates.yaml b/ccs/promptsource/templates/wiki_qa/templates.yaml similarity index 100% rename from elk/promptsource/templates/wiki_qa/templates.yaml rename to ccs/promptsource/templates/wiki_qa/templates.yaml diff --git a/elk/promptsource/templates/wiki_split/templates.yaml b/ccs/promptsource/templates/wiki_split/templates.yaml similarity index 100% rename from elk/promptsource/templates/wiki_split/templates.yaml rename to ccs/promptsource/templates/wiki_split/templates.yaml diff --git a/elk/promptsource/templates/wino_bias/type1_anti/templates.yaml b/ccs/promptsource/templates/wino_bias/type1_anti/templates.yaml similarity index 100% rename from elk/promptsource/templates/wino_bias/type1_anti/templates.yaml rename to ccs/promptsource/templates/wino_bias/type1_anti/templates.yaml diff --git a/elk/promptsource/templates/wino_bias/type1_pro/templates.yaml b/ccs/promptsource/templates/wino_bias/type1_pro/templates.yaml similarity index 100% rename from elk/promptsource/templates/wino_bias/type1_pro/templates.yaml rename to ccs/promptsource/templates/wino_bias/type1_pro/templates.yaml diff --git a/elk/promptsource/templates/wino_bias/type2_anti/templates.yaml b/ccs/promptsource/templates/wino_bias/type2_anti/templates.yaml similarity index 100% rename from elk/promptsource/templates/wino_bias/type2_anti/templates.yaml rename to ccs/promptsource/templates/wino_bias/type2_anti/templates.yaml diff --git a/elk/promptsource/templates/wino_bias/type2_pro/templates.yaml b/ccs/promptsource/templates/wino_bias/type2_pro/templates.yaml similarity index 100% rename from elk/promptsource/templates/wino_bias/type2_pro/templates.yaml rename to ccs/promptsource/templates/wino_bias/type2_pro/templates.yaml diff --git a/elk/promptsource/templates/winograd_wsc/wsc273/templates.yaml b/ccs/promptsource/templates/winograd_wsc/wsc273/templates.yaml similarity index 100% rename from elk/promptsource/templates/winograd_wsc/wsc273/templates.yaml rename to ccs/promptsource/templates/winograd_wsc/wsc273/templates.yaml diff --git a/elk/promptsource/templates/winograd_wsc/wsc285/templates.yaml b/ccs/promptsource/templates/winograd_wsc/wsc285/templates.yaml similarity index 100% rename from elk/promptsource/templates/winograd_wsc/wsc285/templates.yaml rename to ccs/promptsource/templates/winograd_wsc/wsc285/templates.yaml diff --git a/elk/promptsource/templates/winogrande/winogrande_debiased/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_debiased/templates.yaml similarity index 100% rename from elk/promptsource/templates/winogrande/winogrande_debiased/templates.yaml rename to ccs/promptsource/templates/winogrande/winogrande_debiased/templates.yaml diff --git a/elk/promptsource/templates/winogrande/winogrande_l/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_l/templates.yaml similarity index 100% rename from elk/promptsource/templates/winogrande/winogrande_l/templates.yaml rename to ccs/promptsource/templates/winogrande/winogrande_l/templates.yaml diff --git a/elk/promptsource/templates/winogrande/winogrande_m/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_m/templates.yaml similarity index 100% rename from elk/promptsource/templates/winogrande/winogrande_m/templates.yaml rename to ccs/promptsource/templates/winogrande/winogrande_m/templates.yaml diff --git a/elk/promptsource/templates/winogrande/winogrande_s/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_s/templates.yaml similarity index 100% rename from elk/promptsource/templates/winogrande/winogrande_s/templates.yaml rename to ccs/promptsource/templates/winogrande/winogrande_s/templates.yaml diff --git a/elk/promptsource/templates/winogrande/winogrande_xl/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_xl/templates.yaml similarity index 100% rename from elk/promptsource/templates/winogrande/winogrande_xl/templates.yaml rename to ccs/promptsource/templates/winogrande/winogrande_xl/templates.yaml diff --git a/elk/promptsource/templates/winogrande/winogrande_xs/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_xs/templates.yaml similarity index 100% rename from elk/promptsource/templates/winogrande/winogrande_xs/templates.yaml rename to ccs/promptsource/templates/winogrande/winogrande_xs/templates.yaml diff --git a/elk/promptsource/templates/wiqa/templates.yaml b/ccs/promptsource/templates/wiqa/templates.yaml similarity index 100% rename from elk/promptsource/templates/wiqa/templates.yaml rename to ccs/promptsource/templates/wiqa/templates.yaml diff --git a/elk/promptsource/templates/xnli/en/templates.yaml b/ccs/promptsource/templates/xnli/en/templates.yaml similarity index 100% rename from elk/promptsource/templates/xnli/en/templates.yaml rename to ccs/promptsource/templates/xnli/en/templates.yaml diff --git a/elk/promptsource/templates/xquad/xquad.en/templates.yaml b/ccs/promptsource/templates/xquad/xquad.en/templates.yaml similarity index 100% rename from elk/promptsource/templates/xquad/xquad.en/templates.yaml rename to ccs/promptsource/templates/xquad/xquad.en/templates.yaml diff --git a/elk/promptsource/templates/xquad_r/en/templates.yaml b/ccs/promptsource/templates/xquad_r/en/templates.yaml similarity index 100% rename from elk/promptsource/templates/xquad_r/en/templates.yaml rename to ccs/promptsource/templates/xquad_r/en/templates.yaml diff --git a/elk/promptsource/templates/xsum/templates.yaml b/ccs/promptsource/templates/xsum/templates.yaml similarity index 100% rename from elk/promptsource/templates/xsum/templates.yaml rename to ccs/promptsource/templates/xsum/templates.yaml diff --git a/elk/promptsource/templates/yahoo_answers_qa/templates.yaml b/ccs/promptsource/templates/yahoo_answers_qa/templates.yaml similarity index 100% rename from elk/promptsource/templates/yahoo_answers_qa/templates.yaml rename to ccs/promptsource/templates/yahoo_answers_qa/templates.yaml diff --git a/elk/promptsource/templates/yahoo_answers_topics/templates.yaml b/ccs/promptsource/templates/yahoo_answers_topics/templates.yaml similarity index 100% rename from elk/promptsource/templates/yahoo_answers_topics/templates.yaml rename to ccs/promptsource/templates/yahoo_answers_topics/templates.yaml diff --git a/elk/promptsource/templates/yelp_polarity/templates.yaml b/ccs/promptsource/templates/yelp_polarity/templates.yaml similarity index 100% rename from elk/promptsource/templates/yelp_polarity/templates.yaml rename to ccs/promptsource/templates/yelp_polarity/templates.yaml diff --git a/elk/promptsource/templates/yelp_review_full/templates.yaml b/ccs/promptsource/templates/yelp_review_full/templates.yaml similarity index 100% rename from elk/promptsource/templates/yelp_review_full/templates.yaml rename to ccs/promptsource/templates/yelp_review_full/templates.yaml diff --git a/elk/promptsource/templates/zest/templates.yaml b/ccs/promptsource/templates/zest/templates.yaml similarity index 100% rename from elk/promptsource/templates/zest/templates.yaml rename to ccs/promptsource/templates/zest/templates.yaml diff --git a/elk/resources/adjectives.json b/ccs/resources/adjectives.json similarity index 100% rename from elk/resources/adjectives.json rename to ccs/resources/adjectives.json diff --git a/elk/resources/names.json b/ccs/resources/names.json similarity index 100% rename from elk/resources/names.json rename to ccs/resources/names.json diff --git a/elk/run.py b/ccs/run.py similarity index 97% rename from elk/run.py rename to ccs/run.py index fb8903cc..efa8a392 100644 --- a/elk/run.py +++ b/ccs/run.py @@ -20,7 +20,7 @@ from .debug_logging import save_debug_log from .extraction import Extract, extract from .extraction.dataset_name import DatasetDictWithName -from .files import elk_reporter_dir, memorably_named_dir +from .files import ccs_reporter_dir, memorably_named_dir from .utils import ( Color, assert_type, @@ -72,9 +72,9 @@ def execute( if self.out_dir is None: # Save in a memorably-named directory inside of - # ELK_REPORTER_DIR// + # CCS_REPORTER_DIR// ds_name = "+".join(self.data.datasets) - root = elk_reporter_dir() / self.data.model / ds_name + root = ccs_reporter_dir() / self.data.model / ds_name self.out_dir = memorably_named_dir(root) diff --git a/elk/training/__init__.py b/ccs/training/__init__.py similarity index 100% rename from elk/training/__init__.py rename to ccs/training/__init__.py diff --git a/elk/training/burns_norm.py b/ccs/training/burns_norm.py similarity index 100% rename from elk/training/burns_norm.py rename to ccs/training/burns_norm.py diff --git a/elk/training/ccs_reporter.py b/ccs/training/ccs_reporter.py similarity index 99% rename from elk/training/ccs_reporter.py rename to ccs/training/ccs_reporter.py index 7a55a858..ce20fee3 100644 --- a/elk/training/ccs_reporter.py +++ b/ccs/training/ccs_reporter.py @@ -38,7 +38,7 @@ class CcsConfig(FitterConfig): loss: list[str] = field(default_factory=lambda: ["ccs"]) """ The loss function to use. list of strings, each of the form "coef*name", where coef - is a float and name is one of the keys in `elk.training.losses.LOSSES`. + is a float and name is one of the keys in `ccs.training.losses.LOSSES`. Example: `--loss 1.0*consistency_squared 0.5*prompt_var` corresponds to the loss function 1.0*consistency_squared + 0.5*prompt_var. """ diff --git a/elk/training/classifier.py b/ccs/training/classifier.py similarity index 100% rename from elk/training/classifier.py rename to ccs/training/classifier.py diff --git a/elk/training/common.py b/ccs/training/common.py similarity index 100% rename from elk/training/common.py rename to ccs/training/common.py diff --git a/elk/training/eigen_reporter.py b/ccs/training/eigen_reporter.py similarity index 100% rename from elk/training/eigen_reporter.py rename to ccs/training/eigen_reporter.py diff --git a/elk/training/losses.py b/ccs/training/losses.py similarity index 100% rename from elk/training/losses.py rename to ccs/training/losses.py diff --git a/elk/training/platt_scaling.py b/ccs/training/platt_scaling.py similarity index 100% rename from elk/training/platt_scaling.py rename to ccs/training/platt_scaling.py diff --git a/elk/training/supervised.py b/ccs/training/supervised.py similarity index 100% rename from elk/training/supervised.py rename to ccs/training/supervised.py diff --git a/elk/training/sweep.py b/ccs/training/sweep.py similarity index 100% rename from elk/training/sweep.py rename to ccs/training/sweep.py diff --git a/elk/training/train.py b/ccs/training/train.py similarity index 100% rename from elk/training/train.py rename to ccs/training/train.py diff --git a/elk/truncated_eigh.py b/ccs/truncated_eigh.py similarity index 100% rename from elk/truncated_eigh.py rename to ccs/truncated_eigh.py diff --git a/elk/utils/__init__.py b/ccs/utils/__init__.py similarity index 100% rename from elk/utils/__init__.py rename to ccs/utils/__init__.py diff --git a/elk/utils/constants.py b/ccs/utils/constants.py similarity index 100% rename from elk/utils/constants.py rename to ccs/utils/constants.py diff --git a/elk/utils/data_utils.py b/ccs/utils/data_utils.py similarity index 100% rename from elk/utils/data_utils.py rename to ccs/utils/data_utils.py diff --git a/elk/utils/gpu_utils.py b/ccs/utils/gpu_utils.py similarity index 100% rename from elk/utils/gpu_utils.py rename to ccs/utils/gpu_utils.py diff --git a/elk/utils/hf_utils.py b/ccs/utils/hf_utils.py similarity index 100% rename from elk/utils/hf_utils.py rename to ccs/utils/hf_utils.py diff --git a/elk/utils/math_util.py b/ccs/utils/math_util.py similarity index 100% rename from elk/utils/math_util.py rename to ccs/utils/math_util.py diff --git a/elk/utils/pretty.py b/ccs/utils/pretty.py similarity index 100% rename from elk/utils/pretty.py rename to ccs/utils/pretty.py diff --git a/elk/utils/tree_utils.py b/ccs/utils/tree_utils.py similarity index 100% rename from elk/utils/tree_utils.py rename to ccs/utils/tree_utils.py diff --git a/elk/utils/typing.py b/ccs/utils/typing.py similarity index 100% rename from elk/utils/typing.py rename to ccs/utils/typing.py diff --git a/pyproject.toml b/pyproject.toml index cb004269..2c1becb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] -name = "eleuther-elk" -description = "Keeping language models honest by directly eliciting knowledge encoded in their activations" +name = "eleuther-ccs" +description = "Unsupervised methods of eliciting latent knowledge related to Contrast Consistent Search (CCS)" readme = "README.md" requires-python = ">=3.10" keywords = ["nlp", "interpretability", "language-models", "explainable-ai"] @@ -58,20 +58,20 @@ dev = [ ] [project.scripts] -elk = "elk.__main__:run" +ccs = "ccs.__main__:run" [tool.pyright] -include = ["elk*"] +include = ["ccs*"] reportPrivateImportUsage = false [tool.pytest.ini_options] testpaths = ["tests"] [tool.setuptools.package-data] -elk = ["*.json", "*.yaml"] +ccs = ["*.json", "*.yaml"] [tool.setuptools.packages.find] -include = ["elk"] +include = ["ccs"] [tool.ruff] # Enable pycodestyle (`E`), Pyflakes (`F`), and isort (`I`) codes diff --git a/tests/test_burns_norm.py b/tests/test_burns_norm.py index 4620ae0e..09f8fb4a 100644 --- a/tests/test_burns_norm.py +++ b/tests/test_burns_norm.py @@ -1,7 +1,7 @@ import torch from torch import Tensor -from elk.training.burns_norm import BurnsNorm +from ccs.training.burns_norm import BurnsNorm def correct_but_slow_normalization(x_all: Tensor, scale=True) -> Tensor: diff --git a/tests/test_classifier.py b/tests/test_classifier.py index bdc9023d..444c0f0c 100644 --- a/tests/test_classifier.py +++ b/tests/test_classifier.py @@ -2,7 +2,7 @@ from sklearn.datasets import make_classification from sklearn.linear_model import LogisticRegression -from elk.training.classifier import Classifier +from ccs.training.classifier import Classifier @torch.no_grad() diff --git a/tests/test_eigen_reporter.py b/tests/test_eigen_reporter.py index 6303cb03..0edefc77 100644 --- a/tests/test_eigen_reporter.py +++ b/tests/test_eigen_reporter.py @@ -1,7 +1,7 @@ import torch -from elk.training import EigenFitter, EigenFitterConfig -from elk.utils import batch_cov, cov_mean_fused +from ccs.training import EigenFitter, EigenFitterConfig +from ccs.utils import batch_cov, cov_mean_fused def test_eigen_reporter(): diff --git a/tests/test_load_prompts.py b/tests/test_load_prompts.py index 0d309cf5..041c1893 100644 --- a/tests/test_load_prompts.py +++ b/tests/test_load_prompts.py @@ -3,8 +3,8 @@ import pytest -from elk.extraction import Extract, load_prompts -from elk.promptsource.templates import DatasetTemplates +from ccs.extraction import Extract, load_prompts +from ccs.promptsource.templates import DatasetTemplates @pytest.mark.filterwarnings("ignore:Unable to find a decoding function") diff --git a/tests/test_math.py b/tests/test_math.py index 34984d8f..29289d34 100644 --- a/tests/test_math.py +++ b/tests/test_math.py @@ -6,7 +6,7 @@ from hypothesis import given from hypothesis import strategies as st -from elk.utils import batch_cov, cov_mean_fused, stochastic_round_constrained +from ccs.utils import batch_cov, cov_mean_fused, stochastic_round_constrained def test_cov_mean_fused(): diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 3510a302..94b17c0f 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -7,7 +7,7 @@ from sklearn.metrics import roc_auc_score from torch.distributions.normal import Normal -from elk.metrics import accuracy_ci, roc_auc +from ccs.metrics import accuracy_ci, roc_auc def test_auroc_and_acc(): diff --git a/tests/test_samplers.py b/tests/test_samplers.py index 9a1f694e..0857e60d 100644 --- a/tests/test_samplers.py +++ b/tests/test_samplers.py @@ -4,8 +4,8 @@ from datasets import IterableDataset, load_dataset -from elk.extraction import BalancedSampler, FewShotSampler -from elk.utils import assert_type, infer_label_column +from ccs.extraction import BalancedSampler, FewShotSampler +from ccs.utils import assert_type, infer_label_column def test_output_batches_are_balanced(): diff --git a/tests/test_smoke_elicit.py b/tests/test_smoke_elicit.py index bac0f398..0b8b97a0 100644 --- a/tests/test_smoke_elicit.py +++ b/tests/test_smoke_elicit.py @@ -1,8 +1,8 @@ from pathlib import Path -from elk import Extract -from elk.training import CcsConfig, EigenFitterConfig -from elk.training.train import Elicit +from ccs import Extract +from ccs.training import CcsConfig, EigenFitterConfig +from ccs.training.train import Elicit def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path): diff --git a/tests/test_smoke_eval.py b/tests/test_smoke_eval.py index 4efd7112..19e1257c 100644 --- a/tests/test_smoke_eval.py +++ b/tests/test_smoke_eval.py @@ -2,10 +2,10 @@ import pandas as pd -from elk import Extract -from elk.evaluation import Eval -from elk.training import CcsConfig, EigenFitterConfig -from elk.training.train import Elicit +from ccs import Extract +from ccs.evaluation import Eval +from ccs.training import CcsConfig, EigenFitterConfig +from ccs.training.train import Elicit EVAL_EXPECTED_FILES = [ "cfg.yaml", diff --git a/tests/test_truncated_eigh.py b/tests/test_truncated_eigh.py index 5241f1c0..63d06722 100644 --- a/tests/test_truncated_eigh.py +++ b/tests/test_truncated_eigh.py @@ -3,7 +3,7 @@ import torch from scipy.sparse.linalg import eigsh -from elk.truncated_eigh import truncated_eigh +from ccs.truncated_eigh import truncated_eigh def random_symmetric_matrix(n: int, k: int) -> torch.Tensor: diff --git a/tests/test_viz.py b/tests/test_viz.py deleted file mode 100644 index fe3214b0..00000000 --- a/tests/test_viz.py +++ /dev/null @@ -1,31 +0,0 @@ -from pathlib import Path - -import pytest - -from elk.plotting.visualize import SweepVisualization - - -@pytest.fixture -def setup_fs(fs): - test_dir = "/sweep1" - fs.create_dir(test_dir) - fs.create_dir(f"{test_dir}/huggyllama/llama-13b/imdb") - fs.create_file(f"{test_dir}/huggyllama/llama-13b/imdb/eval.csv") - fs.create_dir(f"{test_dir}/huggyllama/llama-12b/news") - fs.create_file(f"{test_dir}/huggyllama/llama-12b/news/eval.csv") - fs.create_file(f"{test_dir}/gpt2-medium/imdb/eval.csv") - - return Path(test_dir) - - -def test_get_model_paths(setup_fs): - test_dir = setup_fs - result = SweepVisualization._get_model_paths(test_dir) - - root = Path(test_dir) - for path in root.rglob("*"): - print(path) - assert len(result) == 3 - assert any([p.name == "llama-13b" for p in result]) - assert any([p.name == "llama-12b" for p in result]) - assert any([p.name == "gpt2-medium" for p in result]) From 2deefc30ecf96672e3349da0280bc807401de58b Mon Sep 17 00:00:00 2001 From: Alex Mallen Date: Fri, 10 Nov 2023 18:26:30 +0000 Subject: [PATCH 4/4] fix type issue --- ccs/metrics/eval.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccs/metrics/eval.py b/ccs/metrics/eval.py index d0e2bf7a..6bd31d1e 100644 --- a/ccs/metrics/eval.py +++ b/ccs/metrics/eval.py @@ -1,5 +1,5 @@ from dataclasses import asdict, dataclass -from typing import Literal +from typing import Any, Literal import torch from einops import repeat @@ -26,7 +26,7 @@ class EvalResult: cal_thresh: float | None """The threshold used to compute the calibrated accuracy.""" - def to_dict(self, prefix: str = "") -> dict[str, float]: + def to_dict(self, prefix: str = "") -> dict[str, Any]: """Convert the result to a dictionary.""" acc_dict = {f"{prefix}acc_{k}": v for k, v in asdict(self.accuracy).items()} cal_acc_dict = (