baidu-ai-search

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python name: Python package on: workflow_dispatch: push: branches: [ "master" ] pull_request: branches: [ "master" ] jobs: Pylint: runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ["3.12.5"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install flake8 pytest python -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9 if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics Python312-CI-Tests: runs-on: ubuntu-latest env: APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }} APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }} BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }} INSTANCE_ID: ${{ secrets.INSTANCE_ID }} DATASET_ID: ${{ secrets.DATASET_ID }} APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }} strategy: fail-fast: false matrix: python-version: ["3.12"] steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Print ENV run: | env ls pwd - name: Git Clone And Checkout Branch run: | fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git" mkdir cicd cd cicd git clone $fork_repo cd app-builder git checkout $GITHUB_HEAD_REF git remote add upstream https://github.com/baidubce/app-builder.git git fetch upstream git remote -v git status # 找到当前分支与 upstream/master 的共同祖先提交 merge_base=$(git merge-base HEAD upstream/master) echo "merge_base=$merge_base" # 比较当前分支与 merge_base 之间的差异 changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base) changed_files_py_sh=$(git diff --name-only --diff-filter=ACMRT $merge_base -- $(find python -type f \( -name '*.py' -o -name '*.sh' \))) echo "发生更改的文件为:" echo "$changed_files" echo "发生更改的py/sh文件为:" if [ -n "$changed_files_py_sh" ]; then export APPBUILDER_PYTHON_TESTS=True echo "$changed_files_py_sh" else export APPBUILDER_PYTHON_TESTS=False echo "没有检测到Python或Shell文件被更改" fi echo "APPBUILDER_PYTHON_TESTS=$APPBUILDER_PYTHON_TESTS" >> $GITHUB_ENV pwd - name: Install dependencies run: | echo $APPBUILDER_PYTHON_TESTS if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Install dependencies部分。" else echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Install dependencies部分..." cd cicd/app-builder pwd sudo apt-get update sudo apt-get install ffmpeg libavcodec-extra python3 -m ensurepip --upgrade python3 -m pip install --upgrade pip python3 -m pip install "setuptools<=76.1.0" python3 -m pip install wheel python3 -m pip install coverage python3 -m pip install diff-cover python3 -m pip install pydub python3 -m pip install SQLAlchemy==2.0.31 python3 -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9 python3 -m pip install opentelemetry-exporter-otlp==1.23.0 opentelemetry-instrumentation==0.44b0 opentelemetry-sdk==1.23.0 opentelemetry-api==1.23.0 python3 -m pip install pandas==2.2.2 python3 -m pip install mcp ; true fi - name: Build whl run: | echo $APPBUILDER_PYTHON_TESTS if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Build whl部分。" else echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Build whl部分..." cd cicd/app-builder pwd python3 setup.py bdist_wheel python3 -m pip uninstall -y appbuilder-sdk python3 -m pip install dist/*.whl fi - name: Test with unittest run: | echo $APPBUILDER_PYTHON_TESTS if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Test with unittest部分。" else echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Test with unittest部分..." cd cicd/app-builder pwd sh python/tests/run_python_test.sh fi Python39-CI-Tests: runs-on: ubuntu-latest env: APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }} APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }} BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }} INSTANCE_ID: ${{ secrets.INSTANCE_ID }} DATASET_ID: ${{ secrets.DATASET_ID }} APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }} strategy: fail-fast: false matrix: python-version: ["3.9"] steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Print ENV run: | env ls pwd - name: Git Clone And Checkout Branch run: | fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git" mkdir cicd cd cicd git clone $fork_repo cd app-builder git checkout $GITHUB_HEAD_REF git remote add upstream https://github.com/baidubce/app-builder.git git fetch upstream git remote -v git status # 找到当前分支与 upstream/master 的共同祖先提交 merge_base=$(git merge-base HEAD upstream/master) echo "merge_base=$merge_base" # 比较当前分支与 merge_base 之间的差异 changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base) changed_files_py_sh=$(git diff --name-only --diff-filter=ACMRT $merge_base -- $(find python -type f \( -name '*.py' -o -name '*.sh' \))) echo "发生更改的文件为:" echo "$changed_files" echo "发生更改的py/sh文件为:" if [ -n "$changed_files_py_sh" ]; then export APPBUILDER_PYTHON_TESTS=True echo "$changed_files_py_sh" else export APPBUILDER_PYTHON_TESTS=False echo "没有检测到Python或Shell文件被更改" fi echo "APPBUILDER_PYTHON_TESTS=$APPBUILDER_PYTHON_TESTS" >> $GITHUB_ENV pwd - name: Install dependencies run: | echo $APPBUILDER_PYTHON_TESTS if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Install dependencies部分。" else echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Install dependencies部分..." cd cicd/app-builder pwd sudo apt-get update sudo apt-get install ffmpeg libavcodec-extra python3 -m ensurepip --upgrade python3 -m pip install --upgrade pip python3 -m pip install "setuptools<=76.1.0" python3 -m pip install wheel python3 -m pip install coverage python3 -m pip install diff-cover python3 -m pip install pydub python3 -m pip install SQLAlchemy==2.0.31 python3 -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9 python3 -m pip install opentelemetry-exporter-otlp==1.23.0 opentelemetry-instrumentation==0.44b0 opentelemetry-sdk==1.23.0 opentelemetry-api==1.23.0 python3 -m pip install pandas==2.2.2 fi - name: Build whl run: | echo $APPBUILDER_PYTHON_TESTS if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Build whl部分。" else echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Build whl部分..." cd cicd/app-builder pwd python3 setup.py bdist_wheel python3 -m pip uninstall -y appbuilder-sdk python3 -m pip install dist/*.whl fi - name: Test with unittest run: | echo $APPBUILDER_PYTHON_TESTS if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Test with unittest部分。" else echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Test with unittest部分..." cd cicd/app-builder pwd sh python/tests/run_python_test.sh fi Go-Test-CI: runs-on: ubuntu-latest env: APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }} APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }} BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }} INSTANCE_ID: ${{ secrets.INSTANCE_ID }} DATASET_ID: ${{ secrets.DATASET_ID }} APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }} APPBUILDER_TOKEN_V3: ${{ secrets.APPBUILDER_TOKEN_V3 }} DATASET_ID_V3: ${{ secrets.DATASET_ID_V3 }} DOCUMENT_ID_V3: ${{ secrets.DOCUMENT_ID_V3 }} strategy: fail-fast: false matrix: go-version: ['1.18'] max-parallel: 2 steps: - name: Checkout code uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Go uses: actions/setup-go@v3 with: go-version: ${{ matrix.go-version }} - name: Git Clone And Checkout Branch run: | fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git" mkdir cicd cd cicd git clone $fork_repo cd app-builder echo "github_head的值为：" echo $GITHUB_HEAD_REF git checkout $GITHUB_HEAD_REF git remote add upstream https://github.com/baidubce/app-builder.git git fetch upstream git remote -v git status # 找到当前分支与 upstream/master 的共同祖先提交 merge_base=$(git merge-base HEAD upstream/master) echo "merge_base=$merge_base" # 比较当前分支与 merge_base 之间的差异 changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base) changed_files_go=$(git diff --name-only --diff-filter=ACMRT $merge_base -- 'go/**') echo "发生更改的文件为:" echo "$changed_files" echo "发生更改的Go相关文件为:" if [ -n "$changed_files_go" ]; then export APPBUILDER_GO_TESTS=True echo "$changed_files_go" else export APPBUILDER_GO_TESTS=False echo "没有检测到Go或Shell文件被更改" fi echo "APPBUILDER_GO_TESTS=$APPBUILDER_GO_TESTS" >> $GITHUB_ENV - name: Install dependencies and tools run: | echo $APPBUILDER_GO_TESTS if [ "$APPBUILDER_GO_TESTS" = "False" ]; then echo "环境变量APPBUILDER_GO_TESTS为False,没有检测到Go文件被更改，跳过Install dependencies and tools部分。" else echo "检测到Go文件被更改，准备启动Install dependencies and tools部分..." cd cicd/app-builder/go/appbuilder go mod tidy go get github.com/axw/gocov/gocov # 安装增量覆盖率工具 # 安装 golangci-lint curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.3 echo "$GOPATH/bin" >> $GITHUB_PATH fi - name: Run tests and calculate incremental coverage run: | echo $APPBUILDER_GO_TESTS if [ "$APPBUILDER_GO_TESTS" = "False" ]; then echo "环境变量APPBUILDER_GO_TESTS为False，没有检测到Go文件被更改，跳过Run tests部分。" else echo "检测到Go文件被更改，准备启动Run tests部分..." cd cicd/app-builder/go # 运行测试并生成覆盖率报告 chmod +x run_go_test.sh ./run_go_test.sh fi shell: bash Java-CI-Tests: runs-on: ubuntu-latest env: APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }} APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }} BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }} INSTANCE_ID: ${{ secrets.INSTANCE_ID }} DATASET_ID: ${{ secrets.DATASET_ID }} APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }} APPBUILDER_TOKEN_V3: ${{ secrets.APPBUILDER_TOKEN_V3 }} DATASET_ID_V3: ${{ secrets.DATASET_ID_V3 }} DOCUMENT_ID_V3: ${{ secrets.DOCUMENT_ID_V3 }} strategy: fail-fast: false matrix: java-version: ['8'] # 使用 Java 11 和 17 作为示例 steps: - name: Checkout code uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up JDK ${{ matrix.java-version }} uses: actions/setup-java@v3 with: java-version: ${{ matrix.java-version }} distribution: 'temurin' - name: Print ENV run: | env ls pwd - name: Git Clone And Checkout Branch run: | fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git" mkdir cicd cd cicd git clone $fork_repo cd app-builder git checkout $GITHUB_HEAD_REF git remote add upstream https://github.com/baidubce/app-builder.git git fetch upstream git remote -v git status # 找到当前分支与 upstream/master 的共同祖先提交 merge_base=$(git merge-base HEAD upstream/master) echo "merge_base=$merge_base" # 比较当前分支与 merge_base 之间的差异 changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base) changed_files_java=$(git diff --name-only --diff-filter=ACMRT $merge_base -- 'java/**') echo "发生更改的文件为:" echo "$changed_files" echo "发生更改的Java相关文件为:" if [ -n "$changed_files_java" ]; then export APPBUILDER_JAVA_TESTS=True echo "$changed_files_java" else export APPBUILDER_JAVA_TESTS=False echo "没有检测到Java文件被更改" fi echo "APPBUILDER_JAVA_TESTS=$APPBUILDER_JAVA_TESTS" >> $GITHUB_ENV pwd - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install gitpython lxml - name: Install dependencies run: | echo $APPBUILDER_JAVA_TESTS if [ "$APPBUILDER_JAVA_TESTS" = "False" ]; then echo "环境变量APPBUILDER_JAVA_TESTS为False,没有检测到Java文件被更改,跳过Install dependencies部分。" else echo "检测到Java文件被更改(根据环境变量APPBUILDER_JAVA_TESTS),准备启动单元测试Install dependencies部分..." cd cicd/app-builder/java mvn clean install -DskipTests fi - name: Run Java tests with Jacoco if: env.APPBUILDER_JAVA_TESTS == 'True' run: | echo "检测到Java文件被更改(根据环境变量APPBUILDER_JAVA_TESTS),准备启动Run Java tests with Jacoco部分..." cd cicd/app-builder/java bash print_coverage.sh