From d199d6c261cabaab32bdcfbefa04671278b53698 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Sat, 27 May 2023 11:51:01 -0400 Subject: [PATCH 1/6] automated testing in github actions --- .github/workflows/tests.yml | 28 +++++++++++++++++++++ requirements-tests.txt | 11 +++++++++ tests/test_prompters.py | 49 +++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 .github/workflows/tests.yml create mode 100644 requirements-tests.txt create mode 100644 tests/test_prompters.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..3543c0b65 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,28 @@ +name: PyTest +on: push + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository code + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + + - name: Install pytest + run: | + pip install --upgrade pytest + + - name: Install dependencies + run: | + pip install -r requirements-tests.txt + + - name: Run tests + run: | + pytest tests/ diff --git a/requirements-tests.txt b/requirements-tests.txt new file mode 100644 index 000000000..e58c194df --- /dev/null +++ b/requirements-tests.txt @@ -0,0 +1,11 @@ +peft @ git+https://github.com/huggingface/peft.git +transformers @ git+https://github.com/huggingface/transformers.git +bitsandbytes>=0.39.0 +attrdict +fire +PyYAML==6.0 +black +datasets +accelerate>=0.19.0 +sentencepiece +wandb diff --git a/tests/test_prompters.py b/tests/test_prompters.py new file mode 100644 index 000000000..1c3c13852 --- /dev/null +++ b/tests/test_prompters.py @@ -0,0 +1,49 @@ +import unittest + +from axolotl.prompters import AlpacaPrompter, PromptStyle + + +class AlpacaPrompterTest(unittest.TestCase): + def test_prompt_style_w_none(self): + prompter = AlpacaPrompter(prompt_style=None) + res = next(prompter.build_prompt("tell me a joke")) + # just testing that it uses instruct style + assert "### Instruction:" in res + + def test_prompt_style_w_instruct(self): + prompter = AlpacaPrompter(prompt_style=PromptStyle.instruct.value) + res = next(prompter.build_prompt("tell me a joke about the following", "alpacas")) + assert "Below is an instruction" in res + assert "### Instruction:" in res + assert "### Input:" in res + assert "alpacas" in res + assert "### Response:" in res + assert "USER:" not in res + assert "ASSISTANT:" not in res + res = next(prompter.build_prompt("tell me a joke about the following")) + assert "Below is an instruction" in res + assert "### Instruction:" in res + assert "### Input:" not in res + assert "### Response:" in res + assert "USER:" not in res + assert "ASSISTANT:" not in res + + def test_prompt_style_w_chat(self): + prompter = AlpacaPrompter(prompt_style=PromptStyle.chat.value) + res = next(prompter.build_prompt("tell me a joke about the following", "alpacas")) + assert "Below is an instruction" in res + assert "### Instruction:" not in res + assert "### Input:" not in res + assert "alpacas" in res + assert "### Response:" not in res + assert "USER:" in res + assert "ASSISTANT:" in res + res = next(prompter.build_prompt("tell me a joke about the following")) + assert "Below is an instruction" in res + assert "### Instruction:" not in res + assert "### Input:" not in res + assert "### Response:" not in res + assert "USER:" in res + assert "ASSISTANT:" in res + + From 403af0b1d78c2f0b6c5e545f6b45869be0f2f13a Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Sat, 27 May 2023 11:58:37 -0400 Subject: [PATCH 2/6] fix path and streamline pip installs --- .github/workflows/tests.yml | 8 ++++---- requirements-tests.txt | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3543c0b65..2dc794be0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,14 +15,14 @@ jobs: with: python-version: "3.9" - - name: Install pytest - run: | - pip install --upgrade pytest - - name: Install dependencies run: | pip install -r requirements-tests.txt + - name: set pythonpath + run: | + echo "PYTHONPATH=src/axolotl" >> $GITHUB_ENV + - name: Run tests run: | pytest tests/ diff --git a/requirements-tests.txt b/requirements-tests.txt index e58c194df..ad24d3588 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,3 +1,5 @@ +pytest +# this is a stripped down version so it doesn't try to install some of the things like xformers or others that may require compilation peft @ git+https://github.com/huggingface/peft.git transformers @ git+https://github.com/huggingface/transformers.git bitsandbytes>=0.39.0 From c29d33352ce133cec92d582143f77203ff73a160 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Sat, 27 May 2023 12:06:23 -0400 Subject: [PATCH 3/6] move python path to same step as tests --- .github/workflows/tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2dc794be0..f8aeb860d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,10 +19,7 @@ jobs: run: | pip install -r requirements-tests.txt - - name: set pythonpath - run: | - echo "PYTHONPATH=src/axolotl" >> $GITHUB_ENV - - name: Run tests run: | + echo "PYTHONPATH=src/axolotl" >> $GITHUB_ENV pytest tests/ From 7f53fd2ab6eda7d92c4192d92225da5dd6f86a40 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Sat, 27 May 2023 12:16:06 -0400 Subject: [PATCH 4/6] alright, just local install it --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f8aeb860d..0d3342e35 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,9 +17,9 @@ jobs: - name: Install dependencies run: | - pip install -r requirements-tests.txt + pip install -e . + pip install pytest - name: Run tests run: | - echo "PYTHONPATH=src/axolotl" >> $GITHUB_ENV pytest tests/ From 72b6ca0d9fd14e29d4caf78192d41fd6d0c6579d Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Sat, 27 May 2023 12:16:54 -0400 Subject: [PATCH 5/6] cache pip --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d3342e35..3cd0d1bd3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: "3.9" - + cache: 'pip' # caching pip dependencies + - name: Install dependencies run: | pip install -e . From a65339228739ee14795b70c959c153e0bcaf5fc2 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Sat, 27 May 2023 12:17:46 -0400 Subject: [PATCH 6/6] use requirements file for tests --- .github/workflows/tests.yml | 4 ++-- requirements-tests.txt | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3cd0d1bd3..8e5836f97 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,11 +15,11 @@ jobs: with: python-version: "3.9" cache: 'pip' # caching pip dependencies - + - name: Install dependencies run: | pip install -e . - pip install pytest + pip install -r requirements-tests.txt - name: Run tests run: | diff --git a/requirements-tests.txt b/requirements-tests.txt index ad24d3588..e079f8a60 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,13 +1 @@ pytest -# this is a stripped down version so it doesn't try to install some of the things like xformers or others that may require compilation -peft @ git+https://github.com/huggingface/peft.git -transformers @ git+https://github.com/huggingface/transformers.git -bitsandbytes>=0.39.0 -attrdict -fire -PyYAML==6.0 -black -datasets -accelerate>=0.19.0 -sentencepiece -wandb