From 6ac4d8b830ad1342dc33b94dd6bd02b850a115a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Fern=C3=A1ndez?= Date: Wed, 31 Jul 2024 14:16:27 +0200 Subject: [PATCH] Add Github build and test actions (#24) Add Github actions: * Build: on pushes to main and dev. * Test: on pushes to main and dev and pull request branches. --- .github/ISSUE_TEMPLATE/bug_report.md | 28 ++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++ .github/pull_request_template.md | 15 ++ .github/workflows/ci.yml | 295 +++++++++++++++++++++- .github/workflows/tests.yml | 90 ++++++- .github_changelog_generator | 5 + docs/analytics/README.md | 1 + reports/Usage Analytics/README.md | 14 +- reports/Usage Analytics/Readme.txt | 1 + 9 files changed, 457 insertions(+), 12 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md create mode 100644 .github_changelog_generator create mode 100644 reports/Usage Analytics/Readme.txt diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..9a3c65c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,28 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: + +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..0f9ea6f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,15 @@ +# Description + +New feature, Bug fixing, or Improvement? +Please include a summary of the change and which issue is fixed. Also include relevant motivation and context. + +## Related issue(s) + +- #X + +## Check list + +- [ ] Related issue / work item is attached +- [ ] Tests are written (if applicable) +- [ ] Documentation is updated (if applicable) +- [ ] Changes are tested diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f3ed8d..0027189 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,299 @@ on: branches: - main - dev - workflow_dispatch: + +env: + Solution_Directory: src\AnalyticsEngine\ + Build_Platform: Any CPU + Build_ProcessorArchitecture: x86 + ClientID: ${{ secrets.APP_CLIENTID }} + ClientSecret: ${{ secrets.APP_CLIENTSECRET }} + TenantGUID: ${{ secrets.APP_TENANTGUID }} + TenantDomain: ${{ secrets.APP_TENANTDOMAIN }} + CognitiveEndpoint: ${{ secrets.APP_COGNITIVEENDPOINT }} + CognitiveKey: ${{ secrets.APP_COGNITIVEKEY}} + CosmosDbTestContainerCurrent: stats + CosmosDbTestContainerHistory: history + CosmosDbTestDatabaseName: UnitTestDevOps + CosmosDb: ${{ secrets.APP_COSMOSDB }} + redis: ${{ secrets.CONNECTIONSTRINGS_REDIS }} + ServiceBus: ${{ secrets.CONNECTIONSTRINGS_SERVICEBUS }} + Storage: ${{ secrets.CONNECTIONSTRINGS_STORAGE }} + SoftwareDownloadURL: ${{ secrets.SOFTWAREDOWNLOADURL }} + StatsApiSecret: ${{ secrets.STATSAPISECRET }} + StatsApiUrl: ${{ secrets.STATSAPIURL }} + jobs: - test: + setup_build: + # This step will calculate the build number with an offset + # so that build numbers will follow the ones from Azure DevOps. + # It will also check that the important folders have been updated. + runs-on: ubuntu-latest + outputs: + build_number: ${{ steps.build_number.outputs.number }} + code_changed: ${{ steps.changes.outputs.code == 'true' }} + steps: + - name: Checkout source + uses: actions/checkout@v4 + - name: Check if code has been modified so a build is needed + id: changes + uses: dorny/paths-filter@v3 + with: + filters: | + code: + - 'src/**' + - 'reports/**' + - name: Calculate build number + id: build_number + run: echo "number=$((${{ vars.BUILD_NUMBER_OFFSET }} + ${{ github.run_number }}))" >> "$GITHUB_OUTPUT" + + build_dotnet: + # Preparation: + # - Get signing certificate + # - Transform configuration files + # - Restore dependencies + # Build: + # - Build .Net projects + # Output: + # - Clean .Net builds + # - Copy PS scripts + # - Zip files + # - Publish artifacts runs-on: windows-latest + needs: setup_build + if: needs.setup_build.outputs.code_changed == 'true' + strategy: + matrix: + configuration: [Release] # [Debug, Release] + env: + BuildId: ${{ needs.setup_build.outputs.build_number }} + BuildLabel: Build ${{ needs.setup_build.outputs.build_number }} + steps: + - name: This is ${{ env.BuildLabel }} + run: echo ${{ env.BuildId }} + - name: Prepare build + id: prep + shell: bash + run: | + mkdir -p "${{ env.Zips_Folder }}" + echo "ZIPS_FOLDER=${{ env.Zips_Folder }}" >> "$GITHUB_OUTPUT" + echo "PFX_PATH=${{ env.Pfx_Path }}" >> "$GITHUB_OUTPUT" + env: + #${{ github.ref_name }} - build ${{ env.BuildId }} + Zips_Folder: ${{ runner.temp }}/zips/ + Pfx_Path: ${{ runner.temp }}/SPOInsightsBinaries.pfx + - name: Decode the PFX + run: | + $pfx_cert_byte = [System.Convert]::FromBase64String("${{ secrets.PFX_BASE64 }}") + [IO.File]::WriteAllBytes("${{ steps.prep.outputs.PFX_PATH }}", $pfx_cert_byte) + - name: Checkout source + uses: actions/checkout@v4 + - name: Setup MSBuild + uses: microsoft/setup-msbuild@v2 + - name: Cache NuGet Packages + id: nuget-packages + uses: actions/cache@v4 + env: + cache-name: nuget-package-cache + with: + path: ~\.nuget\packages + key: ${{ runner.os }}-${{ env.cache-name }} + # Configuration files + - name: Config substitutions + uses: devops-actions/variable-substitution@v1.2 + with: + files: '${{ env.Solution_Directory }}\*\App.Release.config, ${{ env.Solution_Directory }}\Web\Web.Release.config' + - name: Restore the application + run: msbuild ${{ env.Solution_Directory }} -t:Restore ` + -p:Configuration=${{ env.Configuration }} ` + -p:Platform="${{ env.Build_Platform }}" ` + -p:ProcessorArchitecture=${{ env.Build_ProcessorArchitecture }} + env: + Configuration: ${{ matrix.configuration }} + # Projects building + - name: Build WebJob.AppInsightsImporter + run: msbuild ${{ env.Solution_Directory }}\WebJob.AppInsightsImporter\WebJob.AppInsightsImporter.csproj ` + -p:Configuration=${{ env.Configuration }} ` + -p:Platform="${{ env.Build_Platform }}" ` + -p:ProcessorArchitecture=${{ env.Build_ProcessorArchitecture }} ` + -p:OutDir=${{ runner.temp }}\AppInsightsImporter ` + -p:OutputPath=${{ env.Configuration }} ` + -p:AllowedReferenceRelatedFileExtensions=none ` + -p:EmitCompilerGeneratedFiles=false + env: + Configuration: ${{ matrix.configuration }} + - name: Archive AppInsightsImporter + run: | + Remove-Item -Force -Recurse -ErrorAction SilentlyContinue "${{ runner.temp }}\AppInsightsImporter\_PublishedWebsites" + Compress-Archive -Force -Path "${{ runner.temp }}\AppInsightsImporter" ` + -DestinationPath "${{ env.folder }}\AppInsightsImporter.zip" + env: + folder: ${{ steps.prep.outputs.ZIPS_FOLDER }} + - name: Build WebJob.Office365ActivityImporter + run: msbuild ${{ env.Solution_Directory }}\WebJob.Office365ActivityImporter\WebJob.Office365ActivityImporter.csproj ` + -p:Configuration=${{ env.Configuration }} ` + -p:Platform="${{ env.Build_Platform }}" ` + -p:ProcessorArchitecture=${{ env.Build_ProcessorArchitecture }} ` + -p:OutDir=${{ runner.temp }}\Office365ActivityImporter ` + -p:OutputPath=${{ env.Configuration }} ` + -p:AllowedReferenceRelatedFileExtensions=none ` + -p:EmitCompilerGeneratedFiles=false + env: + Configuration: ${{ matrix.configuration }} + - name: Archive Office365ActivityImporter + run: | + Remove-Item -Force -Recurse -ErrorAction SilentlyContinue "${{ runner.temp }}\Office365ActivityImporter\_PublishedWebsites" + Copy-Item -Recurse -Path "src/AnalyticsEngine/WebJob.Office365ActivityImporter/AutomationPS/*" ` + -Destination "${{ runner.temp }}\Office365ActivityImporter\AutomationPS" + Compress-Archive -Force -Path "${{ runner.temp }}\Office365ActivityImporter" ` + -DestinationPath "${{ env.folder }}\Office365ActivityImporter.zip" + env: + folder: ${{ steps.prep.outputs.ZIPS_FOLDER }} + - name: Build Website + run: msbuild ${{ env.Solution_Directory }}\Web\Web.csproj ` + -p:Configuration=${{ env.Configuration }} ` + -p:Platform="${{ env.Build_Platform }}" ` + -p:ProcessorArchitecture=${{ env.Build_ProcessorArchitecture }} ` + -p:OutDir=${{ runner.temp }}\Website ` + -p:OutputPath=${{ env.Configuration }} ` + -p:AllowedReferenceRelatedFileExtensions=none ` + -p:EmitCompilerGeneratedFiles=false + env: + Configuration: ${{ matrix.configuration }} + - name: Archive Website + run: | + Remove-Item -Force -Recurse -ErrorAction SilentlyContinue "${{ runner.temp }}\Website\_PublishedWebsites\Web\bin\Scripts" + Compress-Archive -Force -Path "${{ runner.temp }}\Website\_PublishedWebsites\Web" ` + -DestinationPath "${{ env.folder }}\Website.zip" + env: + folder: ${{ steps.prep.outputs.ZIPS_FOLDER }} + - name: Build Installer + run: | + msbuild ${{ env.Solution_Directory }}\App.ControlPanel\App.ControlPanel.WinForms.csproj ` + -p:Configuration=${{ env.Configuration }} ` + -p:Platform="${{ env.Build_Platform }}" ` + -p:ProcessorArchitecture=${{ env.Build_ProcessorArchitecture }} ` + -p:OutDir=${{ runner.temp }}\ControlPanelApp ` + -p:OutputPath=${{ env.Configuration }} ` + -p:AllowedReferenceRelatedFileExtensions=none ` + -p:EmitCompilerGeneratedFiles=false + env: + Configuration: ${{ matrix.configuration }} + - name: Sign installer + run: | + & 'C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64\signtool.exe' ` + sign /fd SHA256 /a /t http://timestamp.digicert.com /v ` + /f "${{ steps.prep.outputs.PFX_PATH }}" /p "${{ secrets.PFX_KEY}}" ` + "${{ runner.temp }}\ControlPanelApp\AnalyticsInstaller.exe" + env: + Configuration: ${{ matrix.configuration }} + - name: Archive ControlPanelApp + run: Compress-Archive -Force -Path "${{ runner.temp }}\ControlPanelApp" ` + -DestinationPath "${{ env.folder }}\ControlPanelApp.zip" + env: + folder: ${{ steps.prep.outputs.ZIPS_FOLDER }} + - name: Archive Reports + run: | + Get-ChildItem '.\reports\Usage Analytics\*' ` + -Exclude "*_base.pbit" -Include "*.pbit","Readme.txt" | Compress-Archive ` + -Force -DestinationPath "${{ env.folder }}\Analytics_Reports.zip" + env: + folder: ${{ steps.prep.outputs.ZIPS_FOLDER }} + # Wrap up + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + path: ${{ env.folder }} + name: drop_dotnet + if-no-files-found: error + env: + folder: ${{ steps.prep.outputs.ZIPS_FOLDER }} + + - name: Remove the PFX + run: | + Remove-Item -Path "${{ steps.prep.outputs.PFX_PATH }}" + + build_aitracker: + # NPM ai tracker + # Build: + # - Build AI Tracker + # Output: + # - Clean AI Tracker + # - Zip files + # - Publish artifacts + runs-on: ubuntu-latest + needs: setup_build + if: needs.setup_build.outputs.code_changed == 'true' + steps: + - name: Checkout source + uses: actions/checkout@v4 + - name: Build AITracker + run: | + cd src/SPO/AITracker/TypeScript + npm ci + npm run build + cd ../.. + rm -rf AITracker/TypeScript + zip -r AITrackerInstaller.zip AITracker + # Wrap up + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + path: src/SPO/AITrackerInstaller.zip + name: drop_aitracker + if-no-files-found: error + + release: + runs-on: ubuntu-latest + needs: + - setup_build + - build_dotnet + - build_aitracker + permissions: + # Required to create tags in the repo + contents: write + env: + BuildId: ${{ needs.setup_build.outputs.build_number }} + BuildLabel: Build ${{ needs.setup_build.outputs.build_number }} steps: - - name: Checkout + - name: Checkout source uses: actions/checkout@v4 + - name: Install Changelog Generator + run: sudo gem install github_changelog_generator + - name: Generate Changelog + run: | + git branch + if ( ! last_tag="--since-tag $(git describe --tags --abbrev=0)" ); then + last_tag="" + fi + echo $last_tag | xargs github_changelog_generator -u pnp -p Microsoft365-Analytics-Insights \ + --token ${{ github.token }} \ + --release-branch ${{ github.ref_name }} + sed -i '/\[Unreleased\]/,+1d' CHANGELOG.md + - name: Download artifacts + uses: actions/download-artifact@v4 + id: artifacts + - name: Generate release label + run: | + if [ "${{ github.ref_name }}" != "main" ]; then + echo "LABEL=Testing build ${{ env.BuildId }}" >> $GITHUB_ENV + else + echo "LABEL=Stable build ${{ env.BuildId }}" >> $GITHUB_ENV + fi + - name: Create release + uses: ncipollo/release-action@v1.14.0 + id: create_release + env: + GITHUB_TOKEN: ${{ github.token }} + BASE: ${{ steps.artifacts.outputs.download-path }} + with: + draft: ${{ vars.PUBLISH_RELEASES == 'false' }} + prerelease: ${{ github.ref_name == 'dev' || github.ref_name == 'gh-actions' }} + makeLatest: ${{ github.ref_name == 'main' }} + name: ${{ env.LABEL }} + tag: ${{ env.BuildId }} + commit: ${{ github.sha }} + bodyFile: CHANGELOG.md + generateReleaseNotes: true + artifacts: "${{ env.BASE }}/drop_dotnet/*.zip,${{ env.BASE }}/drop_aitracker/*.zip" + artifactContentType: application/zip diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c1c4ae6..0354490 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,9 +11,95 @@ on: types: - opened workflow_dispatch: + +env: + Solution_Directory: src\AnalyticsEngine\Tests.UnitTests + Build_Platform: anycpu + Build_ProcessorArchitecture: x86 + ClientID: ${{ secrets.APP_CLIENTID }} + ClientSecret: ${{ secrets.APP_CLIENTSECRET }} + TenantGUID: ${{ secrets.APP_TENANTGUID }} + TenantDomain: ${{ secrets.APP_TENANTDOMAIN }} + CognitiveEndpoint: ${{ secrets.APP_COGNITIVEENDPOINT }} + CognitiveKey: ${{ secrets.APP_COGNITIVEKEY}} + CosmosDbTestContainerCurrent: stats + CosmosDbTestContainerHistory: history + CosmosDbTestDatabaseName: UnitTestDevOps + CosmosDb: ${{ secrets.APP_COSMOSDB }} + redis: ${{ secrets.CONNECTIONSTRINGS_REDIS }} + ServiceBus: ${{ secrets.CONNECTIONSTRINGS_SERVICEBUS }} + Storage: ${{ secrets.CONNECTIONSTRINGS_STORAGE }} + jobs: - tests: + setup_build: + # Will check that the interesting folders have been updated. + runs-on: ubuntu-latest + outputs: + code_changed: ${{ steps.changes.outputs.code == 'true' }} + steps: + - name: Checkout source + uses: actions/checkout@v4 + - name: Check if the code has been modified so tests must be run + id: changes + uses: dorny/paths-filter@v3 + with: + filters: | + code: + - 'src/**' + unit_tests: + # Tests job + # Preparation: + # - Transform configuration files + # - Restore dependencies + # Build: + # - Build .Net projects + # Test: + # - Run test suite runs-on: windows-latest + needs: setup_build + if: needs.setup_build.outputs.code_changed == 'true' + strategy: + matrix: + configuration: [Release] # [Debug, Release] steps: - - name: Checkout + - name: Checkout source uses: actions/checkout@v4 + - name: Setup MSBuild + uses: microsoft/setup-msbuild@v2 + - name: Cache NuGet Packages + id: nuget-packages + uses: actions/cache@v4 + env: + cache-name: nuget-package-cache + with: + path: ~\.nuget\packages + key: ${{ runner.os }}-${{ env.cache-name }} + - name: Config substitution - Unit tests + uses: microsoft/variable-substitution@v1 + with: + files: '${{ env.Solution_Directory }}\App.Release.config' + - name: Restore the application + run: msbuild ${{ env.Solution_Directory }} -t:Restore ` + -p:Configuration=${{ env.Configuration }} ` + -p:Platform=${{ env.Build_Platform }} ` + -p:ProcessorArchitecture=${{ env.Build_ProcessorArchitecture }} + env: + Configuration: ${{ matrix.configuration }} + - name: Build solution + run: msbuild ${{ env.Solution_Directory }} ` + -p:Configuration=${{ env.Configuration }} ` + -p:Platform=${{ env.Build_Platform }} ` + -p:ProcessorArchitecture=${{ env.Build_ProcessorArchitecture }} + env: + Configuration: ${{ matrix.configuration }} + # Execute all unit tests in the solution + - name: Execute unit tests + uses: jesusfer/vstest-action@main + with: + testAssembly: Tests.UnitTests.dll + searchFolder: ${{ env.Solution_Directory }}\bin\${{ env.Configuration }}\ + runInParallel: true + vstestLocationMethod: location + vstestLocation: C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\Common7\\IDE\\Extensions\\TestPlatform\\vstest.console.exe + env: + Configuration: ${{ matrix.configuration }} diff --git a/.github_changelog_generator b/.github_changelog_generator new file mode 100644 index 0000000..9ada788 --- /dev/null +++ b/.github_changelog_generator @@ -0,0 +1,5 @@ +project=Microsoft365-Analytics-Insights +user=pnp +configure-sections={"Engine":{"prefix":"**Engine updates**","labels":["Engine"]},"Analytics":{"prefix":"**Analytics updates**","labels":["Analytics"]}} +header-label=# What's changed +pr-label=**Merged Pull requests** diff --git a/docs/analytics/README.md b/docs/analytics/README.md index fe0e938..cc92336 100644 --- a/docs/analytics/README.md +++ b/docs/analytics/README.md @@ -1,6 +1,7 @@ # Activity Analytics In this section: + - [Description](#description) - [Dependent Azure resources](#dependent-azure-resources) - [Reports deployment](#reports-deployment) diff --git a/reports/Usage Analytics/README.md b/reports/Usage Analytics/README.md index 2ccb060..3e733db 100644 --- a/reports/Usage Analytics/README.md +++ b/reports/Usage Analytics/README.md @@ -18,10 +18,10 @@ Analytics_Report | Yes | Report for end users 2. Remove end user pages from the report and save it as `Analytics_DataModel.pbix`. 3. Publish the data model to a workspace in the Power BI service. 4. Export the template to `Analytics_DataModel.pbit`. -4. Reopen `Analytics_DataModel_base.pbix`. -5. Un-hide end user pages and remove the rest. -6. Open "Transform data" and remove everything. Close and apply the changes. -7. Remove the Dates table (DAX) from the report. There should not be any data or parameters left in the report. -8. Connect it to the data model published in (3). Use `Get data`->`Power BI semantic models`. -9. Make sure everything works as expected. -10. Save as template to `Analytics_Report.pbit`. \ No newline at end of file +5. Reopen `Analytics_DataModel_base.pbix`. +6. Un-hide end user pages and remove the rest. +7. Open "Transform data" and remove everything. Close and apply the changes. +8. Remove the Dates table (DAX) from the report. There should not be any data or parameters left in the report. +9. Connect it to the data model published in (3). Use `Get data`->`Power BI semantic models`. +10. Make sure everything works as expected. +11. Save as template to `Analytics_Report.pbit`. diff --git a/reports/Usage Analytics/Readme.txt b/reports/Usage Analytics/Readme.txt new file mode 100644 index 0000000..18cf3de --- /dev/null +++ b/reports/Usage Analytics/Readme.txt @@ -0,0 +1 @@ +For deployment instructions of these reports, check the docs: https://github.com/pnp/Microsoft365-Analytics-Insights/blob/dc4f0f09fd36c5d494320de75a00d215742d5ba8/docs/analytics/README.md