From a2d55f4d7a67daa0dd40831e6e7a2ed024991c4a Mon Sep 17 00:00:00 2001 From: kagami Date: Fri, 12 Apr 2024 17:07:35 +0800 Subject: [PATCH 1/9] Init sync-devchat ci --- .github/workflows/sync-devchat.yml | 57 ++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/sync-devchat.yml diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml new file mode 100644 index 0000000..14bc981 --- /dev/null +++ b/.github/workflows/sync-devchat.yml @@ -0,0 +1,57 @@ +name: Sync up devchat packages + +on: + workflow_dispatch: + +jobs: + sync: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: main + + # - name: Create new branch + # run: | + # git checkout -b new-branch + + - name: Install micromamba + run: | + wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba + ./bin/micromamba shell init -s bash -p ~/micromamba + source ~/.bashrc + micromamba create -y -n devchat_no_binary python=3.8 + micromamba activate devchat_no_binary + + - name: Checkout devchat repository to a temporary directory + uses: actions/checkout@v2 + with: + repository: devchat-ai/devchat + path: ./temp-devchat + + - name: Run script + run: | + cd ./temp-devchat + pip list + pip install typer + pip list + + + # - name: Commit changes + # run: | + # git config --local user.email "action@github.com" + # git config --local user.name "GitHub Action" + # git commit -am "Sync up devchat packages" + + # - name: Push changes + # run: | + # git push origin new-branch + + # - name: Create Pull Request + # uses: peter-evans/create-pull-request@v3 + # with: + # title: "Sync up devchat packages" + # branch: "new-branch" + # body: "This PR is auto-generated by GitHub Action. It syncs up the devchat packages." \ No newline at end of file From 09ba772eb88e08601b3d2565e40a9062f4ba5e91 Mon Sep 17 00:00:00 2001 From: kagami Date: Fri, 12 Apr 2024 17:39:36 +0800 Subject: [PATCH 2/9] Update trigger for dev --- .github/workflows/sync-devchat.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml index 14bc981..8e79c33 100644 --- a/.github/workflows/sync-devchat.yml +++ b/.github/workflows/sync-devchat.yml @@ -1,7 +1,10 @@ name: Sync up devchat packages on: - workflow_dispatch: + # workflow_dispatch: + push: + branches: + - ci jobs: sync: From 910dc7d8f34ee953b7e3dfbfd788afca748ade2a Mon Sep 17 00:00:00 2001 From: kagami Date: Fri, 12 Apr 2024 17:43:16 +0800 Subject: [PATCH 3/9] Setup micromamba --- .github/workflows/sync-devchat.yml | 59 +++++++++++++++++++----------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml index 8e79c33..db7f854 100644 --- a/.github/workflows/sync-devchat.yml +++ b/.github/workflows/sync-devchat.yml @@ -11,35 +11,52 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - ref: main + # - name: Checkout code + # uses: actions/checkout@v2 + # with: + # ref: main # - name: Create new branch # run: | # git checkout -b new-branch - - name: Install micromamba - run: | - wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba - ./bin/micromamba shell init -s bash -p ~/micromamba - source ~/.bashrc - micromamba create -y -n devchat_no_binary python=3.8 - micromamba activate devchat_no_binary - - - name: Checkout devchat repository to a temporary directory - uses: actions/checkout@v2 + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v1 with: - repository: devchat-ai/devchat - path: ./temp-devchat + micromamba-version: "1.5.8-0" + init-shell: bash + environment-name: devchat_no_binary + create-args: >- + python=3.8 + # run: | + # "${SHELL}" <(curl -L micro.mamba.pm/install.sh) + # source ~/.bashrc + # eval "$(micromamba shell init --shell bash --root-prefix=~/micromamba 2> /dev/null)" + # micromamba create -y -n devchat_no_binary python=3.8 + # micromamba activate devchat_no_binary - - name: Run script + - name: Check python env + shell: micromamba-shell {0} run: | - cd ./temp-devchat - pip list - pip install typer - pip list + which mamba + mamba --help + python --version + pip --version + which python + which pip + + # - name: Checkout devchat repository to a temporary directory + # uses: actions/checkout@v2 + # with: + # repository: devchat-ai/devchat + # path: ./temp-devchat + + # - name: Run script + # run: | + # cd ./temp-devchat + # pip list + # pip install typer + # pip list # - name: Commit changes From cfc0c46c75c52029f8db82170789451031053e48 Mon Sep 17 00:00:00 2001 From: kagami Date: Fri, 12 Apr 2024 20:42:16 +0800 Subject: [PATCH 4/9] Clone devchat & install dependencies --- .github/workflows/sync-devchat.yml | 51 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml index db7f854..20cdabe 100644 --- a/.github/workflows/sync-devchat.yml +++ b/.github/workflows/sync-devchat.yml @@ -28,35 +28,34 @@ jobs: environment-name: devchat_no_binary create-args: >- python=3.8 - # run: | - # "${SHELL}" <(curl -L micro.mamba.pm/install.sh) - # source ~/.bashrc - # eval "$(micromamba shell init --shell bash --root-prefix=~/micromamba 2> /dev/null)" - # micromamba create -y -n devchat_no_binary python=3.8 - # micromamba activate devchat_no_binary + + - name: Checkout devchat repository to a temporary directory + uses: actions/checkout@v2 + with: + repository: devchat-ai/devchat + path: ./temp-devchat - - name: Check python env + - name: Install devchat dependencies shell: micromamba-shell {0} run: | - which mamba - mamba --help - python --version - pip --version - which python - which pip - - # - name: Checkout devchat repository to a temporary directory - # uses: actions/checkout@v2 - # with: - # repository: devchat-ai/devchat - # path: ./temp-devchat - - # - name: Run script - # run: | - # cd ./temp-devchat - # pip list - # pip install typer - # pip list + echo ">>>>>>>>>> check current path" + pwd + ls + echo ">>>>>>>>>> Check site packages before install" + ls $SITE_PACKAGES + SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") + echo $SITE_PACKAGES + cd ./temp-devchat + pip list + echo ">>>> Installing devchat dependencies" + pip install --no-binary :all: "pydantic<2" + pip install charset-normalizer --no-binary :all: + pip install git+https://github.com/yangbobo2021/tiktoken.git + pip install . + echo ">>>>>>>>>>>> Finish devchat dependencies" + pip list + echo ">>>>>>>>>>>> Check site packages after install" + ls $SITE_PACKAGES # - name: Commit changes From ebbc905e03776afd26ee832bdd85bd1a01fa3fec Mon Sep 17 00:00:00 2001 From: kagami Date: Mon, 15 Apr 2024 09:27:06 +0800 Subject: [PATCH 5/9] Make changes to site-packages --- .github/workflows/sync-devchat.yml | 54 ++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml index 20cdabe..095d0a4 100644 --- a/.github/workflows/sync-devchat.yml +++ b/.github/workflows/sync-devchat.yml @@ -11,14 +11,15 @@ jobs: runs-on: ubuntu-latest steps: - # - name: Checkout code - # uses: actions/checkout@v2 - # with: - # ref: main + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: main - # - name: Create new branch - # run: | - # git checkout -b new-branch + - name: Create new branch + run: | + git checkout -b new-branch + ls - name: Setup micromamba uses: mamba-org/setup-micromamba@v1 @@ -33,7 +34,7 @@ jobs: uses: actions/checkout@v2 with: repository: devchat-ai/devchat - path: ./temp-devchat + path: ./temp/devchat-repo - name: Install devchat dependencies shell: micromamba-shell {0} @@ -42,10 +43,14 @@ jobs: pwd ls echo ">>>>>>>>>> Check site packages before install" - ls $SITE_PACKAGES SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") + ls -al $SITE_PACKAGES echo $SITE_PACKAGES - cd ./temp-devchat + ITEMS_BEFORE=$(mktemp) + ls $SITE_PACKAGES > "$ITEMS_BEFORE" + cat "$ITEMS_BEFORE" + echo ">>>>>>>>>> Start installing..." + cd ./temp/devchat-repo pip list echo ">>>> Installing devchat dependencies" pip install --no-binary :all: "pydantic<2" @@ -56,7 +61,34 @@ jobs: pip list echo ">>>>>>>>>>>> Check site packages after install" ls $SITE_PACKAGES - + ITEMS_AFTER=$(mktemp) + echo ">>>>>>>>>>>>>>>>>>>>>>" + ls $SITE_PACKAGES > "$ITEMS_AFTER" + cat "$ITEMS_AFTER" + echo ">>>>>>>>>>>>>>>>>>>>>>" + ls -al + echo ">>>>>>>>>>>>>>>>>>>>>>Replace the site-packages with the new one" + ITEMS_DIFF=$(comm -13 "$ITEMS_BEFORE" "$ITEMS_AFTER") + echo ">>>>>>>>>>>>>>>>>>>>>> Items diff" + echo $ITEMS_DIFF + echo ">>>>>>>>>>>>>>>>>>>>>>" + cd ../.. + pwd + tree -L 2 + mv ./site-packages ./temp/site-packages-bk + tree -L 2 + mkdir -p ./site-packages + tree -L 2 + echo ">>>>>>>>>>>>>>>>>>>>>> copy the new site-packages" + for item in $ITEMS_DIFF; do + cp -r "$SITE_PACKAGES/$item" ./site-packages + done + echo ">>>>>>>>>>>>>>>>>>>>>> check the new site-packages" + tree -L 2 + # cp -r $SITE_PACKAGES/* ./site-packages/ + # tree -L 1 + + # - name: Commit changes # run: | From 67943102a7496051c794edeaed1edad4c7b9d926 Mon Sep 17 00:00:00 2001 From: kagami Date: Mon, 15 Apr 2024 16:19:13 +0800 Subject: [PATCH 6/9] Commit site-packages changes --- .github/workflows/sync-devchat.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml index 095d0a4..49958e2 100644 --- a/.github/workflows/sync-devchat.yml +++ b/.github/workflows/sync-devchat.yml @@ -16,10 +16,9 @@ jobs: with: ref: main - - name: Create new branch - run: | - git checkout -b new-branch - ls + # - name: Create new branch + # run: | + # git checkout -b update-packages-$(date +'%Y%m%d-%H%M%S') - name: Setup micromamba uses: mamba-org/setup-micromamba@v1 @@ -31,7 +30,7 @@ jobs: python=3.8 - name: Checkout devchat repository to a temporary directory - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: devchat-ai/devchat path: ./temp/devchat-repo @@ -90,11 +89,16 @@ jobs: - # - name: Commit changes - # run: | - # git config --local user.email "action@github.com" - # git config --local user.name "GitHub Action" - # git commit -am "Sync up devchat packages" + - name: Commit and push changes + run: | + git config --local user.email "mingjing@merico.dev" + git config --local user.name "Update-Packages Action" + BRANCH_NAME=update-packages-$(date +'%Y%m%d-%H%M%S') + git checkout -b $BRANCH_NAME + git add ./site-packages + git commit -m "Update devchat site-packages" + git status + git push origin $BRANCH_NAME # - name: Push changes # run: | From 471d5867ba6f31812f2c62c8cd0e73572ff28ffb Mon Sep 17 00:00:00 2001 From: kagami Date: Mon, 15 Apr 2024 17:11:25 +0800 Subject: [PATCH 7/9] Create PR --- .github/workflows/sync-devchat.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml index 49958e2..0dd8d73 100644 --- a/.github/workflows/sync-devchat.yml +++ b/.github/workflows/sync-devchat.yml @@ -9,6 +9,9 @@ on: jobs: sync: runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write steps: - name: Checkout code @@ -84,6 +87,7 @@ jobs: done echo ">>>>>>>>>>>>>>>>>>>>>> check the new site-packages" tree -L 2 + rm -rf ./temp # cp -r $SITE_PACKAGES/* ./site-packages/ # tree -L 1 @@ -99,6 +103,24 @@ jobs: git commit -m "Update devchat site-packages" git status git push origin $BRANCH_NAME + echo "BRANCH_NAME=$BRANCH_NAME" >> "$GITHUB_ENV" + + - name: Create Pull Request + run: | + gh pr create \ + --title "Sync up devchat packages from branch $BRANCH_NAME" \ + --body "This PR is auto-generated by GitHub Action. It syncs up the devchat packages." \ + --base main \ + --head $BRANCH_NAME + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # - name: Create Pull Request + # uses: peter-evans/create-pull-request@v6 + # with: + # title: "Sync up devchat packages from branch ${{ env.BRANCH_NAME }}" + # branch: ${{ env.BRANCH_NAME }} + # base: main + # body: "This PR is auto-generated by GitHub Action. It syncs up the devchat packages from the devchat repository." # - name: Push changes # run: | From 3528d6ae9c8b376f7916cf361bbe3c5a2f4dd10d Mon Sep 17 00:00:00 2001 From: kagami Date: Mon, 15 Apr 2024 21:14:20 +0800 Subject: [PATCH 8/9] Optimize script --- .github/workflows/sync-devchat.yml | 83 +++++++----------------------- 1 file changed, 19 insertions(+), 64 deletions(-) diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml index 0dd8d73..7739093 100644 --- a/.github/workflows/sync-devchat.yml +++ b/.github/workflows/sync-devchat.yml @@ -19,10 +19,6 @@ jobs: with: ref: main - # - name: Create new branch - # run: | - # git checkout -b update-packages-$(date +'%Y%m%d-%H%M%S') - - name: Setup micromamba uses: mamba-org/setup-micromamba@v1 with: @@ -32,103 +28,62 @@ jobs: create-args: >- python=3.8 - - name: Checkout devchat repository to a temporary directory + - name: Clone devchat repo to a temporary directory uses: actions/checkout@v4 with: repository: devchat-ai/devchat path: ./temp/devchat-repo - - name: Install devchat dependencies + - name: Generate the latest site-packages for devchat shell: micromamba-shell {0} run: | - echo ">>>>>>>>>> check current path" - pwd - ls - echo ">>>>>>>>>> Check site packages before install" SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") - ls -al $SITE_PACKAGES - echo $SITE_PACKAGES ITEMS_BEFORE=$(mktemp) ls $SITE_PACKAGES > "$ITEMS_BEFORE" - cat "$ITEMS_BEFORE" - echo ">>>>>>>>>> Start installing..." + echo ">>>>>>>>>> Start installing devchat dependencies..." cd ./temp/devchat-repo - pip list - echo ">>>> Installing devchat dependencies" + DEVCHAT_HEXSHA=$(git rev-parse HEAD | cut -c 1-8) + DEVCHAT_BRANCH=$(git branch --show-current) pip install --no-binary :all: "pydantic<2" pip install charset-normalizer --no-binary :all: pip install git+https://github.com/yangbobo2021/tiktoken.git pip install . - echo ">>>>>>>>>>>> Finish devchat dependencies" + echo ">>>>>>>>>> Finish installing." pip list - echo ">>>>>>>>>>>> Check site packages after install" - ls $SITE_PACKAGES ITEMS_AFTER=$(mktemp) - echo ">>>>>>>>>>>>>>>>>>>>>>" ls $SITE_PACKAGES > "$ITEMS_AFTER" - cat "$ITEMS_AFTER" - echo ">>>>>>>>>>>>>>>>>>>>>>" - ls -al - echo ">>>>>>>>>>>>>>>>>>>>>>Replace the site-packages with the new one" + echo ">>>>>>>>>> Replace the site-packages with the new one" ITEMS_DIFF=$(comm -13 "$ITEMS_BEFORE" "$ITEMS_AFTER") - echo ">>>>>>>>>>>>>>>>>>>>>> Items diff" - echo $ITEMS_DIFF - echo ">>>>>>>>>>>>>>>>>>>>>>" cd ../.. - pwd - tree -L 2 mv ./site-packages ./temp/site-packages-bk - tree -L 2 mkdir -p ./site-packages - tree -L 2 - echo ">>>>>>>>>>>>>>>>>>>>>> copy the new site-packages" for item in $ITEMS_DIFF; do cp -r "$SITE_PACKAGES/$item" ./site-packages done - echo ">>>>>>>>>>>>>>>>>>>>>> check the new site-packages" - tree -L 2 rm -rf ./temp - # cp -r $SITE_PACKAGES/* ./site-packages/ - # tree -L 1 + echo "DEVCHAT_HEXSHA=$DEVCHAT_HEXSHA" >> $GITHUB_ENV + echo "DEVCHAT_BRANCH=$DEVCHAT_BRANCH" >> $GITHUB_ENV - - - name: Commit and push changes + - name: Commit, Push and Create PR run: | git config --local user.email "mingjing@merico.dev" git config --local user.name "Update-Packages Action" - BRANCH_NAME=update-packages-$(date +'%Y%m%d-%H%M%S') + TIMESTAMP=$(date +'%Y%m%d-%H%M%S') + BRANCH_NAME=update-packages-$TIMESTAMP git checkout -b $BRANCH_NAME + git diff --stat git add ./site-packages - git commit -m "Update devchat site-packages" + git commit -m "Sync up site-packages to devchat[$DEVCHAT_BRANCH]($DEVCHAT_HEXSHA)" git status git push origin $BRANCH_NAME - echo "BRANCH_NAME=$BRANCH_NAME" >> "$GITHUB_ENV" - - - name: Create Pull Request - run: | + echo -e "This PR is auto-generated by GitHub Action. \n- devchat branch: \`$DEVCHAT_BRANCH\` \n- devchat hexsha: \`$DEVCHAT_HEXSHA\`" > pr_body + PR_BODY=$(cat pr_body) gh pr create \ - --title "Sync up devchat packages from branch $BRANCH_NAME" \ - --body "This PR is auto-generated by GitHub Action. It syncs up the devchat packages." \ + --title "Sync up site-packages to devchat[$DEVCHAT_BRANCH]($DEVCHAT_HEXSHA) $TIMESTAMP" \ + --body "$PR_BODY" \ + --draft \ --base main \ --head $BRANCH_NAME env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # - name: Create Pull Request - # uses: peter-evans/create-pull-request@v6 - # with: - # title: "Sync up devchat packages from branch ${{ env.BRANCH_NAME }}" - # branch: ${{ env.BRANCH_NAME }} - # base: main - # body: "This PR is auto-generated by GitHub Action. It syncs up the devchat packages from the devchat repository." - - # - name: Push changes - # run: | - # git push origin new-branch - - # - name: Create Pull Request - # uses: peter-evans/create-pull-request@v3 - # with: - # title: "Sync up devchat packages" - # branch: "new-branch" - # body: "This PR is auto-generated by GitHub Action. It syncs up the devchat packages." \ No newline at end of file From 584c9358f0c8275c1aca96344345ed8b40062054 Mon Sep 17 00:00:00 2001 From: kagami Date: Tue, 16 Apr 2024 16:29:36 +0800 Subject: [PATCH 9/9] Make the workflow to be triggered manually --- .github/workflows/sync-devchat.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sync-devchat.yml b/.github/workflows/sync-devchat.yml index 7739093..63024f3 100644 --- a/.github/workflows/sync-devchat.yml +++ b/.github/workflows/sync-devchat.yml @@ -1,10 +1,16 @@ name: Sync up devchat packages on: - # workflow_dispatch: - push: - branches: - - ci + workflow_dispatch: + inputs: + devchat_branch: + description: "The branch of devchat to sync up with" + required: true + default: "main" + type: string + # push: + # branches: + # - ci jobs: sync: @@ -33,6 +39,7 @@ jobs: with: repository: devchat-ai/devchat path: ./temp/devchat-repo + # ref: ${{ github.event.inputs.devchat_branch }} - name: Generate the latest site-packages for devchat shell: micromamba-shell {0}