diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5f3ed8d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,14 @@ +name: ci + +on: + push: + branches: + - main + - dev + workflow_dispatch: +jobs: + test: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 2047f29..d9e2ee3 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ /src/Misc/packages **/.vscode *.tmp +*.tmp.* /src/SPO/Scripts.AITrackerInstaller/AITracker/builduntdeploy.ps1 /src/AnalyticsEngine/Web/Properties/PublishProfiles /src/SPO/Scripts.AITrackerInstaller/DevConfig.json @@ -27,3 +28,4 @@ # Avoid having the pbix files in the repo /reports/Usage Analytics/*.pbix +/src/SPO/AITracker/DevConfig.json diff --git a/docs/Release Notes.md b/docs/Release Notes.md index d308cea..e47cf12 100644 --- a/docs/Release Notes.md +++ b/docs/Release Notes.md @@ -1,4 +1,7 @@ -## master - NEW +## master - build 1400 +Reliability updates for copilot event processing. Previously, SharePoint metadata lookups failed for file specific actions. Also now misc chats are better imported for Copilot actions in Outlook and other app-hosts. + +## master - build 1375 Fixes for SharePoint site aggregation usage stats. When looking up a site ID for a previously imported site (with no ID set), the site lookup save would crash. No SQL schema changes. diff --git a/docs/analytics/README.md b/docs/analytics/README.md index 8767479..fe0e938 100644 --- a/docs/analytics/README.md +++ b/docs/analytics/README.md @@ -192,3 +192,17 @@ The information is displayed in the matrix that can be changed dynamically based 7. At this point, we can publish the report to the same workspace as the previous one. >**Important!** If you want to extend the report to load additional data, create new calculated columns or measures, you should use this report template and not `Analytics_DataModel.pbit`. + +## Additional considerations + +### Audit events + +For audit events to be crawled, the table dbo.OrgUrls must have at least one row. If you are not using SPO Advanced Analytics your table will be empty. + +By default, `Audit.SharePoint` and `Audit.General` are crawled. + +You can add a new row with any content to enable audit logging crawling: + +```SQL +INSERT [dbo].[org_urls] ([url_base]) VALUES ('this-is-here-so-audit-data-can-be-imported'); +``` \ No newline at end of file diff --git a/docs/analytics/data_model.md b/docs/analytics/data_model.md index 75cb934..8c0a9ed 100644 --- a/docs/analytics/data_model.md +++ b/docs/analytics/data_model.md @@ -24,47 +24,82 @@ The date shown in the report will be a Monday, that represents the activity of t The following metrics show the number of activities of this type performed by the user. These are available in the `ActivitiesWeekly` and `ActivitiesWeeklyColumns` tables. +### Activity metrics + **OneDrive** -- Shared Externally -- Shared Internally -- Synced -- Viewed/Edited +* Shared Externally +* Shared Internally +* Synced +* Viewed/Edited **Outlook** -- Emails Read -- Emails Received -- Emails Sent -- Outlook Meetings Created -- Outlook Meetings Interacted +* Emails Read +* Emails Received +* Emails Sent +* Outlook Meetings Created +* Outlook Meetings Interacted **SharePoint** -- Shared Externally -- Shared Internally -- Synced -- Viewed/Edited +* Shared Externally +* Shared Internally +* Synced +* Viewed/Edited **Teams** -- Private Chats -- Calls -- Meetings Attended -- Meetings Organized -- Adhoc Meetings Attended -- Adhoc Meetings Organized -- Scheduled OneTime Meetings Attended -- Scheduled OneTime Meetings Organized -- Scheduled Recurring Meetings Attended -- Scheduled Recurring Meetings Organized -- Audio Duration Seconds -- Video Duration Seconds -- Screenshare Duration Seconds -- Urgent Messages -- Post Messages -- Reply Messages +* Private Chats +* Calls +* Meetings Attended +* Meetings Organized +* Adhoc Meetings Attended +* Adhoc Meetings Organized +* Scheduled OneTime Meetings Attended +* Scheduled OneTime Meetings Organized +* Scheduled Recurring Meetings Attended +* Scheduled Recurring Meetings Organized +* Audio Duration Seconds +* Video Duration Seconds +* Screenshare Duration Seconds +* Urgent Messages +* Post Messages +* Reply Messages **Yammer** -- Liked -- Posted -- Read +* Liked +* Posted +* Read + +**Copilot** + +Hosts where Copilot has been used: +* Assist365 +* Bing +* BashTool +* DevUI +* Excel +* Loop +* M365AdminCenter +* M365App +* Office +* OneNote +* Outlook +* Planner +* PowerPoint +* SharePoint +* Stream +* Teams +* VivaCopilot +* VivaEngage +* VivaGoals +* Whiteboard +* Word + +**Copilot interactions** + +* Number of chat interactions. +* Number of files where there was an interaction (ie. used Copilot to create a slide in PowerPoint). +* Number of meeting where Copilot was used. + +### Usage metrics The following metrics' values are true/false. These are available in the `UsageWeekly` table. @@ -81,25 +116,25 @@ App per platform | | | | **Teams Devices by Platform/OS** -- Used Web -- Used Mobile -- Used Windows -- Used Linux -- Used Mac -- Used Chrome -- Used iOS -- Used Android -- Used WinPhone +* Used Web +* Used Mobile +* Used Windows +* Used Linux +* Used Mac +* Used Chrome +* Used iOS +* Used Android +* Used WinPhone **Yammer Devices by Platoform/OS** -- Used Web -- Used Mobile -- Used Others -- Used iPhone -- Used iPad -- Used Android -- Used WinPhone -- Platform Count +* Used Web +* Used Mobile +* Used Others +* Used iPhone +* Used iPad +* Used Android +* Used WinPhone +* Platform Count ## Raw data @@ -107,28 +142,35 @@ The raw data for the report can be found in the following tables of the database ### Activity tables -- [dbo].[onedrive_user_activity_log] -- [dbo].[outlook_user_activity_log] -- [dbo].[platform_user_activity_log] -- [dbo].[sharepoint_user_activity_log] -- [dbo].[teams_user_activity_log] -- [dbo].[teams_user_device_usage_log] -- [dbo].[yammer_device_activity_log] -- [dbo].[yammer_user_activity_log] +* dbo.onedrive_user_activity_log +* dbo.outlook_user_activity_log +* dbo.platform_user_activity_log +* dbo.sharepoint_user_activity_log +* dbo.teams_user_activity_log +* dbo.teams_user_device_usage_log +* dbo.yammer_device_activity_log +* dbo.yammer_user_activity_log ### User details -- [dbo].[users] -- [dbo].[user_license_type_lookups] -- [dbo].[user_company_name] -- [dbo].[user_country_or_region] -- [dbo].[user_departments] -- [dbo].[user_job_titles] -- [dbo].[user_license_type_lookups] -- [dbo].[user_office_locations] -- [dbo].[user_state_or_province] -- [dbo].[user_usage_locations] -- [dbo].[provinces] -- [dbo].[license_types] -- [dbo].[countries] -- [dbo].[cities] +* dbo.users +* dbo.user_license_type_lookups +* dbo.user_company_name +* dbo.user_country_or_region +* dbo.user_departments +* dbo.user_job_titles +* dbo.user_license_type_lookups +* dbo.user_office_locations +* dbo.user_state_or_province +* dbo.user_usage_locations +* dbo.provinces +* dbo.license_types +* dbo.countries +* dbo.cities + +### Audit events + +* dbo.audit_events +* dbo.event_copilot_chats +* dbo.event_copilot_files +* dbo.event_copilot_meetings diff --git a/docs/install.md b/docs/install.md index a957ebc..88c00e6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -150,7 +150,7 @@ The application should be now created. You need the client ID, secret value, and The runtime service principal must have permissions granted to the activity and Graph API. This is not something granted by default, so needs to be added after registration. -The list of permissions can be found in the [prerequisites docs](prerequisites.md#prerequisite-permissions). +The list of permissions can be found in the [prerequisites docs](prerequisites.md#permissions). In the runtime app registration, add the permissions in the following screenshot. Click "API permissions" and then "Add a permission". diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 8901edb..cda2d06 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -23,7 +23,7 @@ The installer account just needs adding to Azure role-based access control (Azur - Owner of a resource group. - Reader for the subscription. -The runtime account must have the permissions specified below in [Prerequisite Permissions](#prerequisite-permissions) assigned and consented by an administrator. +The runtime account must have the permissions specified below in [Prerequisite Permissions](#permissions) assigned and consented by an administrator. See [here](install.md#create-runtime-application) for more details on how to create these accounts. @@ -35,47 +35,85 @@ The main requirements are around the two Azure AD application registrations used Before attempting to install, verify all the tests pass from the installer. -## Prerequisite Permissions +## Permissions The system requires various permission prerequisites to work. Please verify these are in place before installation. -Element|Description --|- -Installer application/service principal:
**Owner** of solution Azure resource group.
**Read** access to subscription. | The control-panel application creates Azure resources in a single resource-group. The installation service account requires owner permissions of this group (see below). -Office 365 Runtime application/service principal:
**Read access** to Office 365 Activity API.
**Read access** to Graph API.
**Note**: it is possible to run the solution without some of these permissions if an incomplete dataset is acceptable. Where access isn’t granted, the solution will log an error and continue working. | Application permission to read Office 365 data is required & consented from an Azure AD administrator.
**Important**: without an Azure AD administrator’s consent, these permissions will not be active.

**Application permissions**

– Graph API:
**CallRecords.Read.All** – for reading call-record data after calls are made.
**Group.Read.All** – for enumerating Teams data.
**Reports.Read.All** – for reading Teams user usage reports.
**User.Read.All** – for resolving user IDs to email addresses. Required for various processes.
**Organization.Read.All** – for reading SKUs for users at an org level.
**TeamsAppInstallation.ReadForUser.All** – for reading Teams apps users have installed for themselves. Optional if data not needed.
**AppCatalog.Read.All –** for reading apps in the app catalog. Delegated permissions – Graph API:
**ChannelMessage.Read.All** – for deep analytics in selected Teams where enabled (see section 2.4.1).
**Team.ReadBasic.All** – same as above. Application permissions

– Office 365 Management APIs:
**ActivityFeed.Read** – for reading audit data for SharePoint Online events. -Yammer application registration (optional) | Setup needed as defined in [/export · Yammer Developer Site](https://developer.yammer.com/docs/data-export-api) if analytics to Yammer is required. +>**Important:** as permissions are a customer-specific dependency which can take some time to organize & agree, we recommend organizing these permissions first. -User credentials|Description --|- -SharePoint tenant administrator credentials – **only needed if deploying SPO web components**. | Used to install a SharePoint extension add-in into the app-catalog. -SharePoint site-collection administrator credentials – **only needed if deploying SPO web components**. | Usually same user as above; site-collection administrator rights required for target site-collections. -Azure subscription owner credentials. | Needed to grant **owner** rights to the installer service account on the Azure resource-group and **read** rights to the subscription. Once granted, these user credentials should not be needed again. -Windows 10/11 machine with local administrator rights for installer machine. | Needed to run installation PowerShell scripts that rely on the SharePoint Online Management shell (another pre-requisite – see below). +### Installer application/service principal ->**Important:** as permissions are a customer-specific dependency which can take some time to organize & agree, we recommend organizing these permissions first. +The control-panel application creates Azure resources in a single resource-group. The installation service account requires owner permissions of this group (see below). -## Prerequisite Information +* **Owner** of solution Azure resource group +* **Read** access to subscription. -These configuration details should be clarified before deployment. +### Office 365 Runtime application/service principal + +Application permission to read Office 365 data is required & consented from an Azure AD administrator. + +* **Read access** to Office 365 Activity API. +* **Read access** to Graph API. + +>**Note**: it is possible to run the solution without some of these permissions if an incomplete dataset is acceptable. Where access isn’t granted, the solution will log an error and continue working. + +#### Application permissions + +>**Important**: without an Azure AD administrator’s consent, these permissions will not be active. -| Element | Description | -|-|-| -| Azure Subscription Name | Name of the subscription to deploy solution to. | -| Azure Subscription Id | ID of Azure subscription. | -| Office 365 Tenant | Name of the Office 365 Tenant being deployed to. | -| **Optional**: Office 365 Tenant Global SharePoint Administrator credentials | A SharePoint Global Administrator with site-collection permissions to sites being deployed to. | -| OnMicrosoft domain for Office 365 | Tenant name used for Office 365 resources | +API | Permission | Reason +-|-|- +Graph | AppCatalog.Read.All | Reading apps in the Teams app catalog. +Graph | CallRecords.Read.All | Reading call-record data after calls are made. +Graph | Files.Read.All | Reading Copilot event contexts. +Graph | Group.Read.All | Enumerating Teams data. +Graph | OnlineMeetings.Read.All | Reading Copilot event contexts. +Graph | Organization.Read.All | Reading SKUs for users at an org level. +Graph | Reports.Read.All | Reading user and device usage reports. +Graph | TeamsAppInstallation.ReadForUser.All | (Optional) Reading Teams apps users have installed for themselves +Graph | User.Read.All | Resolving user IDs to email addresses. Required for various processes. +Office 365 Management | ActivityFeed.Read | Reading audit data for SharePoint Online events. -## Prerequisite Firewall Rules +(Optional) If you need to access Yammer data, the setup needed is explained in [/export · Yammer Developer Site](https://developer.yammer.com/docs/data-export-api). + +#### Delegated permissions + +API | Permission | Reason +-|-|- +Graph | ChannelMessage.Read.All | For deep analytics in selected Teams where enabled (see section 2.4.1). +Graph | Team.ReadBasic.All | Read basic Teams information. + +### Credentials + +User credentials | Description | When +-|-|- +SharePoint tenant administrator credentials | Used to install a SharePoint extension add-in into the app-catalog | Only if deploying SPO web components +SharePoint site collection administrator credentials | Usually same user as above. Site collection administrator rights required for target site collections | Only if deploying SPO web components +Azure subscription owner credentials. | Needed to grant **owner** rights to the installer service account on the Azure resource-group and **read** rights to the subscription. Once granted, these user credentials should not be needed again. | Initial deployment +Windows 10/11 machine with local administrator rights for installer machine. | Needed to run installation PowerShell scripts that rely on the SharePoint Online Management shell (another pre-requisite – see below). | Initial deployment and updates + +## Deployment Information These configuration details should be clarified before deployment. -Rule | Description +Element | Description -|- -SQL Server - TCP outbound 1433 to your destination SQL server DNS. | Used by installer to create/update database. -FTPS outbound to the app service to be created.
FTPS control connection port: 990
FTPS data connection port: 989, 10001-10300 | Used to publish app-service binaries (web-jobs & website). More info: [https://docs.microsoft.com/en-us/azure/app-service/deploy-ftp?tabs=portal\#troubleshoot-ftp-deployment](https://docs.microsoft.com/en-us/azure/app-service/deploy-ftp?tabs=portal#troubleshoot-ftp-deployment) Ports can be opened only to the FTP domain of the created app service (see below). -**Recommended** – HTTP port 80 to website with no proxy blocking. | Used to detect public IP address, to configure a firewall exception in SQL Server created by setup wizard. If not possible to have setup call the website due to proxy restrictions, add a firewall rule to SQL Server with name “O365 Adv Analytics Setup Rule” so setup will not detect public IP address & create the rule. -Azure Monitor rules | Application Insights needs to be accessible from all client machines that are to be tracked - +Azure Subscription Name | Name of the subscription to deploy solution to. +Azure Subscription Id | ID of Azure subscription. +Office 365 Tenant | Name of the Office 365 Tenant being deployed to. +**Optional**: Office 365 Tenant Global SharePoint Administrator credentials | A SharePoint Global Administrator with site collection permissions to sites being deployed to. +`onmicrosoft.com` domain for Office 365 | Tenant name used for Office 365 resources. + +## Firewall Rules + +These configuration details should be clarified before deployment. + +Service | Rule | Description +-|-|- +SQL Server | TCP outbound 1433 to your destination SQL server DNS. | Used by installer to create/update database. +FTPS outbound to the app service to be created | FTPS control connection port: 990
FTPS data connection port: 989, 10001-10300 | Used to publish app-service binaries (web-jobs & website).
More info: [https://docs.microsoft.com/en-us/azure/app-service/deploy-ftp?tabs=portal\#troubleshoot-ftp-deployment](https://docs.microsoft.com/en-us/azure/app-service/deploy-ftp?tabs=portal#troubleshoot-ftp-deployment) Ports can be opened only to the FTP domain of the created app service (see below). +IP Checker | **Recommended** – HTTP port 80 to website with no proxy blocking | Used to detect public IP address, to configure a firewall exception in SQL Server created by setup wizard. If not possible to have setup call the website due to proxy restrictions, add a firewall rule to SQL Server with name “O365 Adv Analytics Setup Rule” so setup will not detect public IP address & create the rule. +Azure Monitor rules | Several IP addresses | Application Insights needs to be accessible from all client machines that are to be tracked - Get your FTP domain from the app service deployment center (requires the app service to be pre-created): @@ -83,7 +121,7 @@ Get your FTP domain from the app service deployment center (requires the app ser This allows you to limit the outbound ports to a specific DNS name. -## Prerequisite Azure & Office 365 Configuration +## Azure & Office 365 Configuration These configuration details should be verified/configured before deployment. @@ -93,7 +131,7 @@ App registration for **installation account** in target Azure AD where PaaS reso App registration for **Office 365** **runtime account** in target Azure AD used by Office 365 | Client ID + secret for app registration. App registration must have permissions to Office 365 APIs (see below) De-identified information on Office 365: **disabled**. | If “de-identified reports” setting is enabled, the import cannot match Teams or Activity usage reports. This setting isn’t critical but without de-identified reports disabled, activity across all workloads won’t be matched to the same user. For example: SharePoint will show user activity for the right user record, but Teams device activity will only show data for anonymized usernames. The result is this solution will look like x2 users, one anonymized and one real for the same user activity. **Optional:** SharePoint Online Application Catalog | For the tenant being deployed to, the Application catalog needs to be available & configured to upload of the SharePoint Framework Extension. -**Optional**: custom scripts on sites/tenant | Custom scripts need to be enabled on any target SharePoint site-collection for classic page tracking: Modern pages do not require this. +**Optional**: custom scripts on sites/tenant | Custom scripts need to be enabled on any target SharePoint site collection for classic page tracking: Modern pages do not require this. Resource providers enabled on Azure subscription being used by the solution – see | These resource-providers need to be enabled on the subscription used to create the PaaS resources:
Microsoft.Automation
Microsoft.Cache
Microsoft.Insights
Microsoft.KeyVault
Microsoft.OperationalInsights
Microsoft.ServiceBus
Microsoft.Sql
Microsoft.Storage
Microsoft.Web
Microsoft.CognitiveServices (optional; used if sentiment analysis is wanted)
Microsoft.Logic (for Adoptify only) Azure policies to allow public IP/endpoint addresses. | SQL Server PaaS uses public endpoints for databases (with own firewall). Policies that restrict public IP addresses can prevent this required resource from being created, so exceptions must be used if these policies are in place. Storage firewall configuration to allow storage access. | Allow trusted Microsoft services to access the storage account.
Allow access to storage account from installer machine via client IP address/CIDR. diff --git a/reports/Usage Analytics/Analytics_DataModel.pbit b/reports/Usage Analytics/Analytics_DataModel.pbit index 50cec53..ea6baf9 100644 Binary files a/reports/Usage Analytics/Analytics_DataModel.pbit and b/reports/Usage Analytics/Analytics_DataModel.pbit differ diff --git a/reports/Usage Analytics/Analytics_DataModel_base.pbit b/reports/Usage Analytics/Analytics_DataModel_base.pbit new file mode 100644 index 0000000..04932f4 Binary files /dev/null and b/reports/Usage Analytics/Analytics_DataModel_base.pbit differ diff --git a/reports/Usage Analytics/Analytics_Report.pbit b/reports/Usage Analytics/Analytics_Report.pbit index cca4604..23c1a09 100644 Binary files a/reports/Usage Analytics/Analytics_Report.pbit and b/reports/Usage Analytics/Analytics_Report.pbit differ diff --git a/reports/Usage Analytics/README.md b/reports/Usage Analytics/README.md index 259fa22..2ccb060 100644 --- a/reports/Usage Analytics/README.md +++ b/reports/Usage Analytics/README.md @@ -3,3 +3,25 @@ In this folder you can find both reports to deploy for the Activity Analytics feature. Please check [the documentation here](/docs/analytics/README.md) for deployment instructions. + +## Reports + +File | Customizable | Description +-|-|- +Analytics_DataModel_base | No | This is the report that we modify and add new changes, both to the data model and the visuals. The rest of the reports are minified versions of this one. Not part of the release. +Analytics_DataModel | No | Used to deploy the data model only. +Analytics_Report | Yes | Report for end users + +## Report generation steps (Development only) + +1. Open `Analytics_DataModel_base.pbit` and set up the database connection. Save it as `.pbix` for later. +2. Remove end user pages from the report and save it as `Analytics_DataModel.pbix`. +3. Publish the data model to a workspace in the Power BI service. +4. Export the template to `Analytics_DataModel.pbit`. +4. Reopen `Analytics_DataModel_base.pbix`. +5. Un-hide end user pages and remove the rest. +6. Open "Transform data" and remove everything. Close and apply the changes. +7. Remove the Dates table (DAX) from the report. There should not be any data or parameters left in the report. +8. Connect it to the data model published in (3). Use `Get data`->`Power BI semantic models`. +9. Make sure everything works as expected. +10. Save as template to `Analytics_Report.pbit`. \ No newline at end of file diff --git a/reports/Usage Analytics/icons/Copilot_128.png b/reports/Usage Analytics/icons/Copilot_128.png new file mode 100644 index 0000000..8bd6d8f Binary files /dev/null and b/reports/Usage Analytics/icons/Copilot_128.png differ diff --git a/reports/Usage Analytics/icons/Microsoft_Teams_128.png b/reports/Usage Analytics/icons/Microsoft_Teams_128.png new file mode 100644 index 0000000..0bf2231 Binary files /dev/null and b/reports/Usage Analytics/icons/Microsoft_Teams_128.png differ diff --git a/reports/Usage Analytics/icons/OneDrive_128.png b/reports/Usage Analytics/icons/OneDrive_128.png new file mode 100644 index 0000000..9555dca Binary files /dev/null and b/reports/Usage Analytics/icons/OneDrive_128.png differ diff --git a/reports/Usage Analytics/icons/Outlook_128.png b/reports/Usage Analytics/icons/Outlook_128.png new file mode 100644 index 0000000..ca3f777 Binary files /dev/null and b/reports/Usage Analytics/icons/Outlook_128.png differ diff --git a/reports/Usage Analytics/icons/SharePoint_128.png b/reports/Usage Analytics/icons/SharePoint_128.png new file mode 100644 index 0000000..03f22bf Binary files /dev/null and b/reports/Usage Analytics/icons/SharePoint_128.png differ diff --git a/reports/Usage Analytics/icons/Yammer_128.png b/reports/Usage Analytics/icons/Yammer_128.png new file mode 100644 index 0000000..81be6b0 Binary files /dev/null and b/reports/Usage Analytics/icons/Yammer_128.png differ diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/AdoptifyArmInstallTasks.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/AdoptifyArmInstallTasks.cs index 6a8163d..d8ab94e 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/AdoptifyArmInstallTasks.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/AdoptifyArmInstallTasks.cs @@ -2,7 +2,7 @@ using Azure.ResourceManager.Resources; using CloudInstallEngine; using CloudInstallEngine.Azure.InstallTasks; -using Common.DataUtils; +using DataUtils; using Microsoft.Extensions.Logging; using System.Collections.Generic; using System.Threading.Tasks; @@ -20,7 +20,7 @@ public abstract class AdoptifySingleResourceArmInstallTask : GenericArmSingleRes private readonly string _resourceType; private readonly string _json; - protected AdoptifySingleResourceArmInstallTask(string resourceName, TaskConfig configWithTemplateParams, ILogger logger, AzureLocation azureLocation, Dictionary tags, string resourceType, string json) + protected AdoptifySingleResourceArmInstallTask(string resourceName, TaskConfig configWithTemplateParams, ILogger logger, AzureLocation azureLocation, Dictionary tags, string resourceType, string json) : base(configWithTemplateParams, logger, azureLocation, tags) { _resourceName = resourceName; @@ -76,7 +76,7 @@ public override async Task ExecuteTaskReturnResult(object // Todo: add tags to these templates _logger.LogInformation($"Creating API connections"); - var json = new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ApiConnections); + var json = new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ApiConnections); var armParams = _config.FilterParams(ArmParamNames).ToArmParamsObject(new { tagsArray = new { value = Tags } }); var created = await base.ApplyTemplate(armParams, json); @@ -88,7 +88,7 @@ public override async Task ExecuteTaskReturnResult(object public class ProcessDeviceUsageInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessDeviceUsageInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessDeviceUsage)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessDeviceUsage)) { } public override string[] ArmParamNames => new string[] { @@ -99,7 +99,7 @@ public ProcessDeviceUsageInstallTask(string resourceName, TaskConfig config, ILo public class ProcessCallModalityTask : AdoptifySingleLogicAppInstallTask { public ProcessCallModalityTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessCallModality)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessCallModality)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "badgesListId", "userBadgesListId", "userListId", "questsListId", @@ -109,7 +109,7 @@ public ProcessCallModalityTask(string resourceName, TaskConfig config, ILogger l public class ProcessChatsInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessChatsInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessChats)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessChats)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "badgesListId", "userBadgesListId", "userListId", "questsListId", @@ -119,7 +119,7 @@ public ProcessChatsInstallTask(string resourceName, TaskConfig config, ILogger l public class ProcessCountsInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessCountsInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessCounts)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessCounts)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "badgesListId", "userBadgesListId", "userListId", "questsListId", @@ -129,7 +129,7 @@ public ProcessCountsInstallTask(string resourceName, TaskConfig config, ILogger public class ProcessMeetingsInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessMeetingsInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessMeetings)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessMeetings)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "badgesListId", "userBadgesListId", "userListId", "questsListId", @@ -139,7 +139,7 @@ public ProcessMeetingsInstallTask(string resourceName, TaskConfig config, ILogge public class ProcessRedeemedRewardsInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessRedeemedRewardsInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessRedeemedRewards)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessRedeemedRewards)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "userListId", "settingsListId", @@ -149,7 +149,7 @@ public ProcessRedeemedRewardsInstallTask(string resourceName, TaskConfig config, public class ProcessUsageRemindersInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessUsageRemindersInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessUsageReminders)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessUsageReminders)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "userListId", "settingsListId", "rewardsListId", "cardsListId" @@ -158,7 +158,7 @@ public ProcessUsageRemindersInstallTask(string resourceName, TaskConfig config, public class ProcessQuestNotificationsInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessQuestNotificationsInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.QuestNotification)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.QuestNotification)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "userListId", "questsListId", "settingsListId", "cardsListId" @@ -168,7 +168,7 @@ public ProcessQuestNotificationsInstallTask(string resourceName, TaskConfig conf public class ProcessReactionsInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessReactionsInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessReactions)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessReactions)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "badgesListId", "levelsListId", @@ -180,7 +180,7 @@ public ProcessReactionsInstallTask(string resourceName, TaskConfig config, ILogg public class SyncTeamsAppsInstallTask : AdoptifySingleLogicAppInstallTask { public SyncTeamsAppsInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SyncTeamsApps)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.SyncTeamsApps)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "teamsAppsListId", "insightsAppId", "tenantId" @@ -190,7 +190,7 @@ public SyncTeamsAppsInstallTask(string resourceName, TaskConfig config, ILogger public class ProcessTeamsAppsInstallTask : AdoptifySingleLogicAppInstallTask { public ProcessTeamsAppsInstallTask(string resourceName, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) - : base(resourceName, config, logger, azureLocation, tags, new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceStringFromExecutingAssembly(ResourceNameConstants.ProcessTeamsApps)) { } + : base(resourceName, config, logger, azureLocation, tags, new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()).ReadResourceString(ResourceNameConstants.ProcessTeamsApps)) { } public override string[] ArmParamNames => new string[] { "resourceGroupName", "subscriptionId", "location", "adoptifySiteUrl", "questsListId", "questProcessingListId", "userListId", "userQuestsListId", diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/AssetsInstallTask.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/AssetsInstallTask.cs index 40587cd..73f12e6 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/AssetsInstallTask.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/AssetsInstallTask.cs @@ -1,6 +1,6 @@ using App.ControlPanel.Engine.InstallerTasks.Adoptify.Models; using CloudInstallEngine; -using Common.DataUtils; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.SharePoint.Client; using System.IO; @@ -35,7 +35,7 @@ public override async Task ExecuteTask(object contextArg) var listBaseUrl = list.RootFolder.ServerRelativeUrl.TrimStringFromStart(_clientContext.Web.ServerRelativeUrl); var filesUploaded = 0; - var rr = new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()); + var rr = new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()); using (var archive = new ZipArchive(rr.GetAssemblyManifest(ResourceNameConstants.SPAssets), ZipArchiveMode.Read)) { foreach (var item in archive.Entries) diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/ListItemsInstallTask.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/ListItemsInstallTask.cs index 697de78..2c851e9 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/ListItemsInstallTask.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/Adoptify/Tasks/ListItemsInstallTask.cs @@ -1,8 +1,8 @@ using App.ControlPanel.Engine.InstallerTasks.Adoptify.Models; using App.ControlPanel.Engine.SharePointModelBuilder; using CloudInstallEngine; -using Common.DataUtils; using Common.Entities.Installer; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.SharePoint.Client; using System.Threading.Tasks; @@ -34,28 +34,28 @@ public override async Task ExecuteTask(object contextArg) _logger.LogInformation($"Installing default content in language-code '{code}' to {_clientContext.Web.Url}"); - var rr = new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()); + var rr = new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()); var defaultDataQuestsJson = string.Empty; var defaultDataBadgesJson = string.Empty; var defaultDataLevelsJson = string.Empty; - var defaultDataStatsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultStats); + var defaultDataStatsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultStats); var defaultDataSettingsJson = string.Empty; var defaultDataCardsJson = string.Empty; if (code == TargetSolutionConfig.LANG_ESPAÑOL) { - defaultDataSettingsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultSettingsES); - defaultDataQuestsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultQuestsES); - defaultDataLevelsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultLevelsES); - defaultDataBadgesJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultBadgesES); - defaultDataCardsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultCardsES); + defaultDataSettingsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultSettingsES); + defaultDataQuestsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultQuestsES); + defaultDataLevelsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultLevelsES); + defaultDataBadgesJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultBadgesES); + defaultDataCardsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultCardsES); } else { - defaultDataSettingsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultSettingsEN); - defaultDataQuestsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultQuestsEN); - defaultDataLevelsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultLevelsEN); - defaultDataBadgesJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultBadgesEN); - defaultDataCardsJson = rr.ReadResourceStringFromExecutingAssembly(ResourceNameConstants.SPDataDefaultCardsEN); + defaultDataSettingsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultSettingsEN); + defaultDataQuestsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultQuestsEN); + defaultDataLevelsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultLevelsEN); + defaultDataBadgesJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultBadgesEN); + defaultDataCardsJson = rr.ReadResourceString(ResourceNameConstants.SPDataDefaultCardsEN); } // Update field placerholders diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/AppServiceContentInstallJob.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/AppServiceContentInstallJob.cs index 3af69d7..8e167a4 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/AppServiceContentInstallJob.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/AppServiceContentInstallJob.cs @@ -5,11 +5,11 @@ using Azure.ResourceManager.Storage; using CloudInstallEngine; using CloudInstallEngine.Azure.InstallTasks; +using Common.Entities.Installer; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Linq; -using Common.Entities.Installer; namespace App.ControlPanel.Engine.InstallerTasks { diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/InstallAppServiceContentsTask.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/InstallAppServiceContentsTask.cs index d638145..6ff98fd 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/InstallAppServiceContentsTask.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/InstallAppServiceContentsTask.cs @@ -24,7 +24,7 @@ public class InstallAppServiceContentsTask : InstallTaskInAzResourceGroup tags) + public InstallAppServiceContentsTask(InstallerFtpConfig ftpConfig, TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) : base(config, logger, azureLocation, tags) { _ftpConfig = ftpConfig; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/RunbookCreateOrUpdateTasks.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/RunbookCreateOrUpdateTasks.cs index bed258b..86c9ba6 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/RunbookCreateOrUpdateTasks.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/RunbookCreateOrUpdateTasks.cs @@ -34,7 +34,7 @@ public override Task ExecuteTaskReturnResult(obje } public class ProfilingScriptAggregationStatusPSRunbookUploadTask : RunbookUploadTask { - public ProfilingScriptAggregationStatusPSRunbookUploadTask(TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) + public ProfilingScriptAggregationStatusPSRunbookUploadTask(TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) : base(config, logger, azureLocation, tags) { } @@ -53,7 +53,7 @@ public override Task ExecuteTaskReturnResult(obje } public class ProfilingScriptDatabaseMaintenancePSRunbookUploadTask : RunbookUploadTask { - public ProfilingScriptDatabaseMaintenancePSRunbookUploadTask(TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) + public ProfilingScriptDatabaseMaintenancePSRunbookUploadTask(TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) : base(config, logger, azureLocation, tags) { } diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/SoftwarePackageDownloadTasks.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/SoftwarePackageDownloadTasks.cs index c2cacaa..ad75deb 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/SoftwarePackageDownloadTasks.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/SoftwarePackageDownloadTasks.cs @@ -3,7 +3,7 @@ using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using CloudInstallEngine; -using Common.DataUtils; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.IO; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/RunbooksInstallJob.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/RunbooksInstallJob.cs index 1981ccd..8d5ddbf 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/RunbooksInstallJob.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/RunbooksInstallJob.cs @@ -1,6 +1,5 @@ using App.ControlPanel.Engine.InstallerTasks.JobTasks; using App.ControlPanel.Engine.Models; -using Azure.Core; using Azure.ResourceManager.Automation; using Azure.ResourceManager.Resources; using Azure.ResourceManager.Storage; @@ -18,7 +17,7 @@ namespace App.ControlPanel.Engine.InstallerTasks public class RunbooksInstallJob : InstallJobInContainerJob { public RunbooksInstallJob(ILogger logger, SolutionInstallConfig config, SubscriptionResource subscription, StorageAccountResource storageAccount, - AutomationAccountResource automationAccount) + AutomationAccountResource automationAccount) : base(logger, new ResourceGroupContainerLoader(TaskConfig.GetConfigForName(config.ResourceGroupName), logger, subscription, config.AzureLocation, config.Tags.ToDictionary())) { // Upload automation PS files to storage account diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Base64Serialisable.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Base64Serialisable.cs index 65889ce..fcb77d1 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Base64Serialisable.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Base64Serialisable.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using Newtonsoft.Json; using System; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Config/AppRegistrationCredentials.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Config/AppRegistrationCredentials.cs index a64fb3a..fc57751 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Config/AppRegistrationCredentials.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Config/AppRegistrationCredentials.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System.Collections.Generic; namespace App.ControlPanel.Engine.Entities diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Config/SolutionInstallConfig.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Config/SolutionInstallConfig.cs index 7f6396f..6cb5fb4 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Config/SolutionInstallConfig.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/Models/Config/SolutionInstallConfig.cs @@ -1,8 +1,8 @@ using App.ControlPanel.Engine.Entities; using App.ControlPanel.Engine.Models; using Azure.Core; -using Common.DataUtils; using Common.Entities.Installer; +using DataUtils; using Newtonsoft.Json; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SharePointModelBuilder/SiteBuilder.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SharePointModelBuilder/SiteBuilder.cs index 950be64..b570935 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SharePointModelBuilder/SiteBuilder.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SharePointModelBuilder/SiteBuilder.cs @@ -1,6 +1,6 @@ using App.ControlPanel.Engine.SharePointModelBuilder.ValueLookups; using CloudInstallEngine.Models; -using Common.DataUtils; +using DataUtils; using Microsoft.SharePoint.Client; using Newtonsoft.Json.Linq; using OfficeDevPnP.Core.Framework.Provisioning.Model; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SharePointModelBuilder/ValueLookups/AbstractListItemValueLookup.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SharePointModelBuilder/ValueLookups/AbstractListItemValueLookup.cs index 1eafa34..6d64e82 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SharePointModelBuilder/ValueLookups/AbstractListItemValueLookup.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SharePointModelBuilder/ValueLookups/AbstractListItemValueLookup.cs @@ -1,5 +1,5 @@ using CloudInstallEngine.Models; -using Common.DataUtils; +using DataUtils; using System; using System.Text.Json; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SiteTrackerInstaller/SiteAITrackerInstaller.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SiteTrackerInstaller/SiteAITrackerInstaller.cs index 9f4597c..17d6737 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SiteTrackerInstaller/SiteAITrackerInstaller.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/SPO/SiteTrackerInstaller/SiteAITrackerInstaller.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/SolutionInstallVerifier.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/SolutionInstallVerifier.cs index 67e4681..4d45264 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/SolutionInstallVerifier.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/SolutionInstallVerifier.cs @@ -7,8 +7,8 @@ using Azure.ResourceManager.Resources; using Azure.ResourceManager.Sql; using CloudInstallEngine.Azure; -using Common.DataUtils; -using Common.DataUtils.Http; +using DataUtils; +using DataUtils.Http; using FluentFTP.Exceptions; using Microsoft.Extensions.Logging; using System; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/SqlExtentions/Profiling-03-CreateSchema.sql b/src/AnalyticsEngine/App.ControlPanel.Engine/SqlExtentions/Profiling-03-CreateSchema.sql index 50352da..abdc0ed 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/SqlExtentions/Profiling-03-CreateSchema.sql +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/SqlExtentions/Profiling-03-CreateSchema.sql @@ -1,6 +1,9 @@ -SET ANSI_NULLS ON +/* tsqllint-disable warning set-transaction-isolation-level */ + +SET ANSI_NULLS ON; GO -SET QUOTED_IDENTIFIER ON + +SET QUOTED_IDENTIFIER ON; GO -- ============================================ @@ -8,33 +11,96 @@ GO -- ===== Create indexes in dbo tables ===== -- ===== ===== -- ============================================ -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.teams_user_activity_log')) - CREATE INDEX IX_date ON [dbo].[teams_user_activity_log] (date); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.teams_user_activity_log') +) +BEGIN + CREATE INDEX IX_date ON dbo.teams_user_activity_log ("date"); +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.onedrive_user_activity_log')) - CREATE INDEX IX_date ON [dbo].[onedrive_user_activity_log] (date); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.onedrive_user_activity_log') +) +BEGIN + CREATE INDEX IX_date ON dbo.onedrive_user_activity_log ("date"); +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.outlook_user_activity_log')) - CREATE INDEX IX_date ON [dbo].[outlook_user_activity_log] (date); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.outlook_user_activity_log') +) +BEGIN + CREATE INDEX IX_date ON dbo.outlook_user_activity_log ("date"); +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.sharepoint_user_activity_log')) - CREATE INDEX IX_date ON [dbo].[sharepoint_user_activity_log] (date); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.sharepoint_user_activity_log') +) +BEGIN + CREATE INDEX IX_date ON dbo.sharepoint_user_activity_log ("date"); +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.yammer_user_activity_log')) - CREATE INDEX IX_date ON [dbo].[yammer_user_activity_log] (date); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.yammer_user_activity_log') +) +BEGIN + CREATE INDEX IX_date ON dbo.yammer_user_activity_log ("date"); +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_users_account_enabled' AND object_id = OBJECT_ID('dbo.users')) - CREATE NONCLUSTERED INDEX IX_users_account_enabled - ON [dbo].[users] (account_enabled) INCLUDE (azure_ad_id); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_users_account_enabled' AND object_id = OBJECT_ID('dbo.users') +) +BEGIN + CREATE NONCLUSTERED INDEX IX_users_account_enabled ON dbo.users (account_enabled) + INCLUDE (azure_ad_id); +END +GO + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.teams_user_device_usage_log') +) +BEGIN + CREATE INDEX IX_date ON dbo.teams_user_device_usage_log ("date"); +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.teams_user_device_usage_log')) - CREATE INDEX IX_date ON [dbo].[teams_user_device_usage_log] (date); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.platform_user_activity_log') +) +BEGIN + CREATE INDEX IX_date ON dbo.platform_user_activity_log ("date"); +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.platform_user_activity_log')) - CREATE INDEX IX_date ON [dbo].[platform_user_activity_log] (date); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.yammer_device_activity_log') +) +BEGIN + CREATE INDEX IX_date ON dbo.yammer_device_activity_log ("date"); +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('dbo.yammer_device_activity_log')) - CREATE INDEX IX_date ON [dbo].[yammer_device_activity_log] (date); + +IF NOT EXISTS ( + SELECT name FROM sys.indexes + WHERE name = N'IX_user_id' AND object_id = OBJECT_ID('dbo.user_license_type_lookups') +) +BEGIN + CREATE NONCLUSTERED INDEX IX_user_id ON dbo.user_license_type_lookups ("user_id"); +END GO -- ===================================== @@ -44,29 +110,76 @@ GO -- ===== ===== -- ===================================== -IF OBJECT_ID(N'usp_GetErrorInfo') IS NOT NULL DROP PROCEDURE usp_GetErrorInfo; +IF OBJECT_ID(N'usp_GetErrorInfo') IS NOT NULL +BEGIN + DROP PROCEDURE usp_GetErrorInfo; +END + GO -IF OBJECT_ID(N'profiling.usp_CompileDaily') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileDaily]; +IF OBJECT_ID(N'profiling.usp_CompileDaily') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileDaily; +END GO -IF OBJECT_ID(N'profiling.usp_Version') IS NOT NULL DROP PROCEDURE [profiling].[usp_Version]; + +IF OBJECT_ID(N'profiling.usp_Version') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_Version; +END GO -IF OBJECT_ID(N'profiling.udf_GetLimitDate') IS NOT NULL DROP FUNCTION [profiling].[udf_GetLimitDate]; + +IF OBJECT_ID(N'profiling.udf_GetLimitDate') IS NOT NULL +BEGIN + DROP FUNCTION profiling.udf_GetLimitDate; +END GO -IF OBJECT_ID(N'profiling.tvf_ActivitiesBetweenDates') IS NOT NULL DROP FUNCTION [profiling].[tvf_ActivitiesBetweenDates]; + +IF OBJECT_ID(N'profiling.tvf_ActivitiesBetweenDates') IS NOT NULL +BEGIN + DROP FUNCTION profiling.tvf_ActivitiesBetweenDates; +END GO -IF OBJECT_ID(N'profiling.tvf_DuplicatedUsers') IS NOT NULL DROP FUNCTION [profiling].[tvf_DuplicatedUsers]; + +IF OBJECT_ID(N'profiling.tvf_DuplicatedUsers') IS NOT NULL +BEGIN + DROP FUNCTION profiling.tvf_DuplicatedUsers; +END GO -IF OBJECT_ID(N'profiling.tvf_Version') IS NOT NULL DROP PROCEDURE [profiling].[tvf_Version]; + +IF OBJECT_ID(N'profiling.tvf_Version') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.tvf_Version; +END GO -IF OBJECT_ID(N'profiling.Activities') IS NOT NULL DROP TABLE [profiling].[Activities]; + +IF OBJECT_ID(N'profiling.Activities') IS NOT NULL +BEGIN + DROP TABLE profiling.Activities; +END GO -IF OBJECT_ID(N'profiling.ActivitiesDaily') IS NOT NULL DROP TABLE [profiling].[ActivitiesDaily]; + +IF OBJECT_ID(N'profiling.ActivitiesDaily') IS NOT NULL +BEGIN + DROP TABLE profiling.ActivitiesDaily; +END GO -IF OBJECT_ID(N'profiling.usp_CompileWeek') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileWeek]; + +IF OBJECT_ID(N'profiling.usp_CompileWeek') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileWeek; +END GO -IF OBJECT_ID(N'profiling.usp_CompileWeekColumns') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileWeekColumns]; + +IF OBJECT_ID(N'profiling.usp_CompileWeekColumns') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileWeekColumns; +END GO -IF OBJECT_ID(N'profiling.usp_CompileWeekRows') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileWeekRows]; + +IF OBJECT_ID(N'profiling.usp_CompileWeekRows') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileWeekRows; +END GO -- ================================= @@ -75,8 +188,14 @@ GO -- ===== ===== -- ================================= -IF NOT EXISTS (SELECT name FROM sys.schemas WHERE name = N'profiling') - EXEC('CREATE SCHEMA [profiling] AUTHORIZATION [dbo]'); +IF NOT EXISTS ( + SELECT name + FROM sys.schemas + WHERE name = N'profiling' +) +BEGIN + EXEC ('CREATE SCHEMA profiling AUTHORIZATION dbo;'); +END GO -- ===================== @@ -88,39 +207,63 @@ GO -- ============================================= -- View to get users worth having in the reports -- ============================================= -IF OBJECT_ID(N'profiling.users') IS NOT NULL DROP VIEW [profiling].[users]; + +IF OBJECT_ID(N'profiling.users') IS NOT NULL +BEGIN + DROP VIEW profiling.users; +END GO -CREATE VIEW [profiling].[users] +CREATE VIEW profiling.users AS WITH cte_license_count AS ( - SELECT user_id, COUNT(id) license_count - FROM [dbo].[user_license_type_lookups] - GROUP BY user_id + SELECT + "user_id", + COUNT(id) AS license_count + FROM dbo.user_license_type_lookups + GROUP BY "user_id" ), cte_enabled_users AS ( SELECT id FROM dbo.users - WHERE account_enabled = 1 - AND azure_ad_id IS NOT NULL AND azure_ad_id <> '' + WHERE account_enabled = 1 AND azure_ad_id IS NOT NULL AND azure_ad_id <> '' ) - SELECT eu.id, user_name, azure_ad_id, mail, - office_location_id, usage_location_id, department_id, job_title_id, - postalcode, company_name_id, state_or_province_id, manager_id, country_or_region_id, + SELECT + eu.id, + user_name, + azure_ad_id, + mail, + office_location_id, + usage_location_id, + department_id, + job_title_id, + postalcode, + company_name_id, + state_or_province_id, + manager_id, + country_or_region_id, license_count - FROM cte_license_count l - INNER JOIN cte_enabled_users eu ON eu.id = l.user_id - INNER JOIN dbo.users u ON u.id = eu.id; + FROM + cte_license_count AS l + INNER JOIN cte_enabled_users AS eu ON eu.id = l."user_id" + INNER JOIN dbo.users AS u ON u.id = eu.id; GO -- =============================== -- View to get unique postal codes -- =============================== -IF OBJECT_ID('profiling.user_PostalCodes') IS NOT NULL DROP VIEW [profiling].[user_PostalCodes]; + +IF OBJECT_ID('profiling.user_PostalCodes') IS NOT NULL +BEGIN + DROP VIEW profiling.user_PostalCodes; +END GO -CREATE VIEW [profiling].[user_PostalCodes] AS SELECT DISTINCT [postalcode] FROM [profiling].[users]; +CREATE VIEW profiling.user_PostalCodes +AS + SELECT DISTINCT postalcode + FROM profiling.users; GO -- ====================== @@ -130,234 +273,440 @@ GO -- ====================== IF OBJECT_ID(N'profiling.TraceLogs') IS NULL - CREATE TABLE [profiling].[TraceLogs] ( - [Id] BIGINT NOT NULL IDENTITY(1,1) PRIMARY KEY, - [Datetime] DATETIME NOT NULL, - [Message] NVARCHAR(500) NOT NULL +BEGIN + CREATE TABLE profiling.TraceLogs + ( + Id BIGINT NOT NULL IDENTITY(1, 1) PRIMARY KEY, + "Datetime" DATETIME NOT NULL, + Message NVARCHAR(500) NOT NULL ); +END +GO + +-- ============================================= +-- Aggregates all possible weekly analytics data +-- ============================================= + +IF OBJECT_ID(N'profiling.usp_Trace') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_Trace; +END +GO + +CREATE PROCEDURE profiling.usp_Trace +( + @Message NVARCHAR(500), + @p1 NVARCHAR(50) = NULL, + @p2 NVARCHAR(50) = NULL, + @p3 NVARCHAR(50) = NULL, + @p4 NVARCHAR(50) = NULL, + @p5 NVARCHAR(50) = NULL +) +AS +BEGIN + DECLARE @FormattedMessage NVARCHAR(MAX); + SELECT @FormattedMessage = FORMATMESSAGE(@Message, CAST(@p1 AS NVARCHAR(50)), CAST(@p2 AS NVARCHAR(50)), CAST(@p3 AS NVARCHAR(50)), CAST(@p4 AS NVARCHAR(50)), CAST(@p5 AS NVARCHAR(50))); + + INSERT INTO profiling.TraceLogs ("Datetime", Message) + SELECT GETDATE(), @FormattedMessage; +END; GO -- ====================================================== -- Weekly aggregated activity data per user. Data in rows -- ====================================================== + IF OBJECT_ID(N'profiling.ActivitiesWeekly') IS NULL - CREATE TABLE [profiling].[ActivitiesWeekly] ( - [user_id] BIGINT NOT NULL, - [MetricDate] DATE NOT NULL, - [Metric] VARCHAR(250) NOT NULL, - [Sum] INT NOT NULL, - CONSTRAINT [PK_ActivitiesWeekly] PRIMARY KEY CLUSTERED ([user_id] ASC, [MetricDate] ASC, [Metric] ASC) WITH ( - STATISTICS_NORECOMPUTE = OFF, - IGNORE_DUP_KEY = OFF, - OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF - ) ON [PRIMARY] +BEGIN + CREATE TABLE profiling.ActivitiesWeekly + ( + "user_id" BIGINT NOT NULL, + MetricDate DATE NOT NULL, + Metric VARCHAR(250) NOT NULL, + Sum INT NOT NULL, + CONSTRAINT PK_ActivitiesWeekly + PRIMARY KEY CLUSTERED ("user_id" ASC, MetricDate ASC, Metric ASC) + WITH ( + STATISTICS_NORECOMPUTE = OFF + ,IGNORE_DUP_KEY = OFF + -- ,OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF + ) ); +END; GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_MetricDate' AND object_id = OBJECT_ID('profiling.ActivitiesWeekly')) - CREATE INDEX IX_MetricDate ON [profiling].[ActivitiesWeekly] (MetricDate); +IF NOT EXISTS ( + SELECT name + FROM sys.indexes + WHERE object_id = OBJECT_ID('profiling.ActivitiesWeekly') + AND name = N'IX_MetricDate' +) +BEGIN + CREATE INDEX IX_MetricDate ON profiling.ActivitiesWeekly (MetricDate); +END GO -- ========================================================= -- Weekly aggregated activity data per user. Data in columns -- ========================================================= + IF OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') IS NULL - CREATE TABLE [profiling].[ActivitiesWeeklyColumns] ( - user_id BIGINT, - [date] DATE, - [OneDrive Viewed/Edited] BIGINT NOT NULL DEFAULT 0, - [OneDrive Synced] BIGINT NOT NULL DEFAULT 0, - [OneDrive Shared Internally] BIGINT NOT NULL DEFAULT 0, - [OneDrive Shared Externally] BIGINT NOT NULL DEFAULT 0, - [Emails Sent] BIGINT NOT NULL DEFAULT 0, - [Emails Received] BIGINT NOT NULL DEFAULT 0, - [Emails Read] BIGINT NOT NULL DEFAULT 0, - [Outlook Meetings Created] BIGINT NOT NULL DEFAULT 0, - [Outlook Meetings Interacted] BIGINT NOT NULL DEFAULT 0, - [SPO Viewed/Edited] BIGINT NOT NULL DEFAULT 0, - [SPO Synced] BIGINT NOT NULL DEFAULT 0, - [SPO Shared Internally] BIGINT NOT NULL DEFAULT 0, - [SPO Shared Externally] BIGINT NOT NULL DEFAULT 0, - [Teams Private Chats] BIGINT NOT NULL DEFAULT 0, - [Teams Team Chats] BIGINT NOT NULL DEFAULT 0, - [Teams Calls] BIGINT NOT NULL DEFAULT 0, - [Teams Meetings] BIGINT NOT NULL DEFAULT 0, - [Teams Meetings Attended] BIGINT NOT NULL DEFAULT 0, - [Teams Meetings Organized] BIGINT NOT NULL DEFAULT 0, - [Yammer Posted] BIGINT NOT NULL DEFAULT 0, - [Yammer Read] BIGINT NOT NULL DEFAULT 0, - [Yammer Liked] BIGINT NOT NULL DEFAULT 0, - CONSTRAINT [PK_ActivitiesWeeklyColumns] PRIMARY KEY CLUSTERED ([user_id] ASC, [date] ASC) WITH ( - STATISTICS_NORECOMPUTE = OFF, - IGNORE_DUP_KEY = OFF, - OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF - ) ON [PRIMARY] +BEGIN + CREATE TABLE profiling.ActivitiesWeeklyColumns + ( + "user_id" BIGINT, + "date" DATE, + "OneDrive Viewed/Edited" BIGINT NOT NULL DEFAULT 0, + "OneDrive Synced" BIGINT NOT NULL DEFAULT 0, + "OneDrive Shared Internally" BIGINT NOT NULL DEFAULT 0, + "OneDrive Shared Externally" BIGINT NOT NULL DEFAULT 0, + "Emails Sent" BIGINT NOT NULL DEFAULT 0, + "Emails Received" BIGINT NOT NULL DEFAULT 0, + "Emails Read" BIGINT NOT NULL DEFAULT 0, + "Outlook Meetings Created" BIGINT NOT NULL DEFAULT 0, + "Outlook Meetings Interacted" BIGINT NOT NULL DEFAULT 0, + "SPO Viewed/Edited" BIGINT NOT NULL DEFAULT 0, + "SPO Synced" BIGINT NOT NULL DEFAULT 0, + "SPO Shared Internally" BIGINT NOT NULL DEFAULT 0, + "SPO Shared Externally" BIGINT NOT NULL DEFAULT 0, + "Teams Private Chats" BIGINT NOT NULL DEFAULT 0, + "Teams Team Chats" BIGINT NOT NULL DEFAULT 0, + "Teams Calls" BIGINT NOT NULL DEFAULT 0, + "Teams Meetings" BIGINT NOT NULL DEFAULT 0, + "Teams Meetings Attended" BIGINT NOT NULL DEFAULT 0, + "Teams Meetings Organized" BIGINT NOT NULL DEFAULT 0, + "Yammer Posted" BIGINT NOT NULL DEFAULT 0, + "Yammer Read" BIGINT NOT NULL DEFAULT 0, + "Yammer Liked" BIGINT NOT NULL DEFAULT 0, + "Copilot Chats" BIGINT NOT NULL DEFAULT 0, + "Copilot Meetings" BIGINT NOT NULL DEFAULT 0, + "Copilot Files" BIGINT NOT NULL DEFAULT 0, + "Copilot App Assist365" BIGINT NOT NULL DEFAULT 0, + "Copilot App Bing" BIGINT NOT NULL DEFAULT 0, + "Copilot App BashTool" BIGINT NOT NULL DEFAULT 0, + "Copilot App DevUI" BIGINT NOT NULL DEFAULT 0, + "Copilot App Excel" BIGINT NOT NULL DEFAULT 0, + "Copilot App Loop" BIGINT NOT NULL DEFAULT 0, + "Copilot App M365AdminCenter" BIGINT NOT NULL DEFAULT 0, + "Copilot App M365App" BIGINT NOT NULL DEFAULT 0, + "Copilot App Office" BIGINT NOT NULL DEFAULT 0, + "Copilot App OneNote" BIGINT NOT NULL DEFAULT 0, + "Copilot App Outlook" BIGINT NOT NULL DEFAULT 0, + "Copilot App Planner" BIGINT NOT NULL DEFAULT 0, + "Copilot App PowerPoint" BIGINT NOT NULL DEFAULT 0, + "Copilot App SharePoint" BIGINT NOT NULL DEFAULT 0, + "Copilot App Stream" BIGINT NOT NULL DEFAULT 0, + "Copilot App Teams" BIGINT NOT NULL DEFAULT 0, + "Copilot App VivaCopilot" BIGINT NOT NULL DEFAULT 0, + "Copilot App VivaEngage" BIGINT NOT NULL DEFAULT 0, + "Copilot App VivaGoals" BIGINT NOT NULL DEFAULT 0, + "Copilot App Whiteboard" BIGINT NOT NULL DEFAULT 0, + "Copilot App Word" BIGINT NOT NULL DEFAULT 0, + + CONSTRAINT PK_ActivitiesWeeklyColumns + PRIMARY KEY CLUSTERED ("user_id" ASC, "date" ASC) + WITH ( + STATISTICS_NORECOMPUTE = OFF + ,IGNORE_DUP_KEY = OFF + -- ,OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF + ) ); +END GO -- Add possibly missing default contraints -IF NOT EXISTS(SELECT OBJECT_NAME(OBJECT_ID) FROM sys.objects WHERE type_desc LIKE 'DEFAULT_CONSTRAINT' AND SCHEMA_NAME(schema_id) = 'profiling' AND OBJECT_NAME(parent_object_id) = 'ActivitiesWeeklyColumns') - ALTER TABLE [profiling].[ActivitiesWeeklyColumns] +IF NOT EXISTS ( + SELECT OBJECT_NAME(OBJECT_ID) + FROM sys.objects + WHERE + type_desc LIKE 'DEFAULT_CONSTRAINT' + AND SCHEMA_NAME(SCHEMA_ID) = 'profiling' + AND OBJECT_NAME(parent_object_id) = 'ActivitiesWeeklyColumns' +) +BEGIN + ALTER TABLE profiling.ActivitiesWeeklyColumns + ADD + CONSTRAINT df_onedrive_viewed_edited DEFAULT 0 FOR "OneDrive Viewed/Edited", + CONSTRAINT df_onedrive_synced DEFAULT 0 FOR "OneDrive Synced", + CONSTRAINT df_onedrive_shared_internally DEFAULT 0 FOR "OneDrive Shared Internally", + CONSTRAINT df_onedrive_shared_externally DEFAULT 0 FOR "OneDrive Shared Externally", + CONSTRAINT df_emails_sent DEFAULT 0 FOR "Emails Sent", + CONSTRAINT df_emails_received DEFAULT 0 FOR "Emails Received", + CONSTRAINT df_emails_read DEFAULT 0 FOR "Emails Read", + CONSTRAINT df_outlook_meetings_created DEFAULT 0 FOR "Outlook Meetings Created", + CONSTRAINT df_outlook_meetings_interacted DEFAULT 0 FOR "Outlook Meetings Interacted", + CONSTRAINT df_spo_viewed_edited DEFAULT 0 FOR "SPO Viewed/Edited", + CONSTRAINT df_spo_synced DEFAULT 0 FOR "SPO Synced", + CONSTRAINT df_spo_shared_internally DEFAULT 0 FOR "SPO Shared Internally", + CONSTRAINT df_spo_shared_externally DEFAULT 0 FOR "SPO Shared Externally", + CONSTRAINT df_teams_private_chats DEFAULT 0 FOR "Teams Private Chats", + CONSTRAINT df_teams_team_chats DEFAULT 0 FOR "Teams Team Chats", + CONSTRAINT df_teams_calls DEFAULT 0 FOR "Teams Calls", + CONSTRAINT df_teams_meetings DEFAULT 0 FOR "Teams Meetings", + CONSTRAINT df_teams_meetings_attended DEFAULT 0 FOR "Teams Meetings Attended", + CONSTRAINT df_teams_meetings_organized DEFAULT 0 FOR "Teams Meetings Organized", + CONSTRAINT df_yammer_posted DEFAULT 0 FOR "Yammer Posted", + CONSTRAINT df_yammer_read DEFAULT 0 FOR "Yammer Read", + CONSTRAINT df_yammer_liked DEFAULT 0 FOR "Yammer Liked"; +END +GO + +-- Add new Teams columns +IF NOT EXISTS ( + SELECT 1 + FROM sys.columns + WHERE object_id = OBJECT_ID('profiling.ActivitiesWeeklyColumns') + AND name = 'Teams Adhoc Meetings Attended' +) +BEGIN + ALTER TABLE profiling.ActivitiesWeeklyColumns ADD - CONSTRAINT df_onedrive_viewed_edited DEFAULT 0 FOR [OneDrive Viewed/Edited], - CONSTRAINT df_onedrive_synced DEFAULT 0 FOR [OneDrive Synced], - CONSTRAINT df_onedrive_shared_internally DEFAULT 0 FOR [OneDrive Shared Internally], - CONSTRAINT df_onedrive_shared_externally DEFAULT 0 FOR [OneDrive Shared Externally], - CONSTRAINT df_emails_sent DEFAULT 0 FOR [Emails Sent], - CONSTRAINT df_emails_received DEFAULT 0 FOR [Emails Received], - CONSTRAINT df_emails_read DEFAULT 0 FOR [Emails Read], - CONSTRAINT df_outlook_meetings_created DEFAULT 0 FOR [Outlook Meetings Created], - CONSTRAINT df_outlook_meetings_interacted DEFAULT 0 FOR [Outlook Meetings Interacted], - CONSTRAINT df_spo_viewed_edited DEFAULT 0 FOR [SPO Viewed/Edited], - CONSTRAINT df_spo_synced DEFAULT 0 FOR [SPO Synced], - CONSTRAINT df_spo_shared_internally DEFAULT 0 FOR [SPO Shared Internally], - CONSTRAINT df_spo_shared_externally DEFAULT 0 FOR [SPO Shared Externally], - CONSTRAINT df_teams_private_chats DEFAULT 0 FOR [Teams Private Chats], - CONSTRAINT df_teams_team_chats DEFAULT 0 FOR [Teams Team Chats], - CONSTRAINT df_teams_calls DEFAULT 0 FOR [Teams Calls], - CONSTRAINT df_teams_meetings DEFAULT 0 FOR [Teams Meetings], - CONSTRAINT df_teams_meetings_attended DEFAULT 0 FOR [Teams Meetings Attended], - CONSTRAINT df_teams_meetings_organized DEFAULT 0 FOR [Teams Meetings Organized], - CONSTRAINT df_yammer_posted DEFAULT 0 FOR [Yammer Posted], - CONSTRAINT df_yammer_read DEFAULT 0 FOR [Yammer Read], - CONSTRAINT df_yammer_liked DEFAULT 0 FOR [Yammer Liked]; -GO - --- Add new metric columns -IF NOT EXISTS(SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 'ActivitiesWeeklyColumns' AND TABLE_SCHEMA = 'profiling' AND COLUMN_NAME = 'Teams Adhoc Meetings Attended') - ALTER TABLE [profiling].[ActivitiesWeeklyColumns] + "Teams Adhoc Meetings Attended" BIGINT NOT NULL DEFAULT 0, + "Teams Adhoc Meetings Organized" BIGINT NOT NULL DEFAULT 0, + "Teams Scheduled Onetime Meetings Attended" BIGINT NOT NULL DEFAULT 0, + "Teams Scheduled Onetime Meetings Organized" BIGINT NOT NULL DEFAULT 0, + "Teams Scheduled Recurring Meetings Attended" BIGINT NOT NULL DEFAULT 0, + "Teams Scheduled Recurring Meetings Organized" BIGINT NOT NULL DEFAULT 0, + "Teams Audio Duration Seconds" BIGINT NOT NULL DEFAULT 0, + "Teams Video Duration Seconds" BIGINT NOT NULL DEFAULT 0, + "Teams Screenshare Duration Seconds" BIGINT NOT NULL DEFAULT 0, + "Teams Post Messages" BIGINT NOT NULL DEFAULT 0, + "Teams Reply Messages" BIGINT NOT NULL DEFAULT 0, + "Teams Urgent Messages" BIGINT NOT NULL DEFAULT 0; +END +GO + +-- Add new Copilot columns +IF NOT EXISTS ( + SELECT 1 + FROM sys.columns + WHERE object_id = OBJECT_ID('profiling.ActivitiesWeeklyColumns') + AND name = 'Copilot Chats' +) +BEGIN + ALTER TABLE + profiling.ActivitiesWeeklyColumns ADD - [Teams Adhoc Meetings Attended] BIGINT NOT NULL DEFAULT 0 - ,[Teams Adhoc Meetings Organized] BIGINT NOT NULL DEFAULT 0 - ,[Teams Scheduled Onetime Meetings Attended] BIGINT NOT NULL DEFAULT 0 - ,[Teams Scheduled Onetime Meetings Organized] BIGINT NOT NULL DEFAULT 0 - ,[Teams Scheduled Recurring Meetings Attended] BIGINT NOT NULL DEFAULT 0 - ,[Teams Scheduled Recurring Meetings Organized] BIGINT NOT NULL DEFAULT 0 - ,[Teams Audio Duration Seconds] BIGINT NOT NULL DEFAULT 0 - ,[Teams Video Duration Seconds] BIGINT NOT NULL DEFAULT 0 - ,[Teams Screenshare Duration Seconds] BIGINT NOT NULL DEFAULT 0 - ,[Teams Post Messages] BIGINT NOT NULL DEFAULT 0 - ,[Teams Reply Messages] BIGINT NOT NULL DEFAULT 0 - ,[Teams Urgent Messages] BIGINT NOT NULL DEFAULT 0; + "Copilot Chats" BIGINT NOT NULL DEFAULT 0, + "Copilot Meetings" BIGINT NOT NULL DEFAULT 0, + "Copilot Files" BIGINT NOT NULL DEFAULT 0, + "Copilot App Assist365" BIGINT NOT NULL DEFAULT 0, + "Copilot App Bing" BIGINT NOT NULL DEFAULT 0, + "Copilot App BashTool" BIGINT NOT NULL DEFAULT 0, + "Copilot App DevUI" BIGINT NOT NULL DEFAULT 0, + "Copilot App Excel" BIGINT NOT NULL DEFAULT 0, + "Copilot App Loop" BIGINT NOT NULL DEFAULT 0, + "Copilot App M365AdminCenter" BIGINT NOT NULL DEFAULT 0, + "Copilot App M365App" BIGINT NOT NULL DEFAULT 0, + "Copilot App Office" BIGINT NOT NULL DEFAULT 0, + "Copilot App OneNote" BIGINT NOT NULL DEFAULT 0, + "Copilot App Outlook" BIGINT NOT NULL DEFAULT 0, + "Copilot App Planner" BIGINT NOT NULL DEFAULT 0, + "Copilot App PowerPoint" BIGINT NOT NULL DEFAULT 0, + "Copilot App SharePoint" BIGINT NOT NULL DEFAULT 0, + "Copilot App Stream" BIGINT NOT NULL DEFAULT 0, + "Copilot App Teams" BIGINT NOT NULL DEFAULT 0, + "Copilot App VivaCopilot" BIGINT NOT NULL DEFAULT 0, + "Copilot App VivaEngage" BIGINT NOT NULL DEFAULT 0, + "Copilot App VivaGoals" BIGINT NOT NULL DEFAULT 0, + "Copilot App Whiteboard" BIGINT NOT NULL DEFAULT 0, + "Copilot App Word" BIGINT NOT NULL DEFAULT 0; +END GO -IF NOT EXISTS (SELECT name FROM sys.indexes WHERE name = N'IX_date' AND object_id = OBJECT_ID('profiling.ActivitiesWeeklyColumns')) - CREATE INDEX IX_date ON [profiling].[ActivitiesWeeklyColumns] (date); +IF NOT EXISTS ( + SELECT name + FROM sys.indexes + WHERE object_id = OBJECT_ID('profiling.ActivitiesWeeklyColumns') + AND name = N'IX_date' +) +BEGIN + CREATE INDEX IX_date ON profiling.ActivitiesWeeklyColumns ("date"); +END GO -- Remove Yammer device columns. First constraints, then the columns -IF EXISTS (select object_id from sys.columns WHERE name = 'Yammer Platform Count' AND object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns')) +IF EXISTS ( + SELECT 1 + FROM sys.columns + WHERE object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND name = 'Yammer Platform Count' +) BEGIN DECLARE @name NVARCHAR(200); - SELECT @name=obj.name FROM sys.objects obj JOIN sys.columns cols ON obj.object_id = cols.default_object_id - WHERE cols.name = 'Yammer Platform Count' AND cols.object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns'); - EXEC ('ALTER TABLE [profiling].[ActivitiesWeeklyColumns] DROP CONSTRAINT ' + @name); + SELECT @name = obj.name + FROM sys.objects AS obj + JOIN sys.columns AS cols ON obj.object_id = cols.default_object_id + WHERE cols.object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND cols.name = 'Yammer Platform Count'; + EXEC ('ALTER TABLE profiling.ActivitiesWeeklyColumns DROP CONSTRAINT ' + @name); END GO -IF EXISTS (select object_id from sys.columns WHERE name = 'Yammer Platform Count' AND object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns')) - ALTER TABLE [profiling].[ActivitiesWeeklyColumns] DROP COLUMN [Yammer Platform Count]; + +IF EXISTS ( + SELECT object_id + FROM sys.columns + WHERE object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND name = 'Yammer Platform Count' +) +BEGIN + ALTER TABLE profiling.ActivitiesWeeklyColumns DROP COLUMN "Yammer Platform Count"; +END GO -IF EXISTS (select object_id from sys.columns WHERE name = 'Yammer Used Web' AND object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns')) +IF EXISTS ( + SELECT object_id + FROM sys.columns + WHERE object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND name = 'Yammer Used Web' +) BEGIN DECLARE @name NVARCHAR(200); - SELECT @name=obj.name FROM sys.objects obj JOIN sys.columns cols ON obj.object_id = cols.default_object_id - WHERE cols.name = 'Yammer Used Web' AND cols.object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns'); - EXEC ('ALTER TABLE [profiling].[ActivitiesWeeklyColumns] DROP CONSTRAINT ' + @name); + SELECT @name = obj.name + FROM sys.objects AS obj + JOIN sys.columns AS cols ON obj.object_id = cols.default_object_id + WHERE cols.object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND cols.name = 'Yammer Used Web'; + EXEC ('ALTER TABLE profiling.ActivitiesWeeklyColumns DROP CONSTRAINT ' + @name); END GO -IF EXISTS (select object_id from sys.columns WHERE name = 'Yammer Used Web' AND object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns')) - ALTER TABLE [profiling].[ActivitiesWeeklyColumns] DROP COLUMN [Yammer Used Web]; + +IF EXISTS ( + SELECT object_id + FROM sys.columns + WHERE object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND name = 'Yammer Used Web' +) +BEGIN + ALTER TABLE profiling.ActivitiesWeeklyColumns DROP COLUMN "Yammer Used Web"; +END GO -IF EXISTS (select object_id from sys.columns WHERE name = 'Yammer Used Mobile' AND object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns')) +IF EXISTS ( + SELECT object_id + FROM sys.columns + WHERE object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND name = 'Yammer Used Mobile' +) BEGIN DECLARE @name NVARCHAR(200); - SELECT @name=obj.name FROM sys.objects obj JOIN sys.columns cols ON obj.object_id = cols.default_object_id - WHERE cols.name = 'Yammer Used Mobile' AND cols.object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns'); - EXEC ('ALTER TABLE [profiling].[ActivitiesWeeklyColumns] DROP CONSTRAINT ' + @name); + SELECT @name = obj.name + FROM sys.objects AS obj + JOIN sys.columns AS cols ON obj.object_id = cols.default_object_id + WHERE cols.object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND cols.name = 'Yammer Used Mobile'; + EXEC ('ALTER TABLE profiling.ActivitiesWeeklyColumns DROP CONSTRAINT ' + @name); END GO -IF EXISTS (select object_id from sys.columns WHERE name = 'Yammer Used Mobile' AND object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns')) - ALTER TABLE [profiling].[ActivitiesWeeklyColumns] DROP COLUMN [Yammer Used Mobile]; + +IF EXISTS ( + SELECT object_id + FROM sys.columns + WHERE object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND name = 'Yammer Used Mobile' +) +BEGIN + ALTER TABLE profiling.ActivitiesWeeklyColumns DROP COLUMN "Yammer Used Mobile"; +END GO -IF EXISTS (select object_id from sys.columns WHERE name = 'Yammer Used Others' AND object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns')) + +IF EXISTS ( + SELECT object_id + FROM sys.columns + WHERE object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND name = 'Yammer Used Others' +) BEGIN DECLARE @name NVARCHAR(200); - SELECT @name=obj.name FROM sys.objects obj JOIN sys.columns cols ON obj.object_id = cols.default_object_id - WHERE cols.name = 'Yammer Used Others' AND cols.object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns'); - EXEC ('ALTER TABLE [profiling].[ActivitiesWeeklyColumns] DROP CONSTRAINT ' + @name); + SELECT @name = obj.name + FROM sys.objects AS obj + JOIN sys.columns AS cols ON obj.object_id = cols.default_object_id + WHERE cols.object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND cols.name = 'Yammer Used Others'; + EXEC ('ALTER TABLE profiling.ActivitiesWeeklyColumns DROP CONSTRAINT ' + @name); END GO -IF EXISTS (select object_id from sys.columns WHERE name = 'Yammer Used Others' AND object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns')) - ALTER TABLE [profiling].[ActivitiesWeeklyColumns] DROP COLUMN [Yammer Used Others]; + +IF EXISTS ( + SELECT object_id + FROM sys.columns + WHERE object_id = OBJECT_ID(N'profiling.ActivitiesWeeklyColumns') + AND name = 'Yammer Used Others' +) +BEGIN + ALTER TABLE profiling.ActivitiesWeeklyColumns DROP COLUMN "Yammer Used Others"; +END GO -- ====================================================== -- Weekly aggregated usage data per user. Data in columns -- ====================================================== IF OBJECT_ID(N'profiling.UsageWeekly') IS NULL - CREATE TABLE [profiling].[UsageWeekly] ( - [user_id] [int] NOT NULL, - [date] [date] NOT NULL, - -- Teams devices - [Teams Used Web] [bit] NOT NULL DEFAULT 0, - [Teams Used Mac] [bit] NOT NULL DEFAULT 0, - [Teams Used Windows] [bit] NOT NULL DEFAULT 0, - [Teams Used Linux] [bit] NOT NULL DEFAULT 0, - [Teams Used Chrome OS] [bit] NOT NULL DEFAULT 0, - [Teams Used Mobile] [bit] NOT NULL DEFAULT 0, - [Teams Used WinPhone] [bit] NOT NULL DEFAULT 0, - [Teams Used iOS] [bit] NOT NULL DEFAULT 0, - [Teams Used Android] [bit] NOT NULL DEFAULT 0, - -- M365 app - [Office Windows] [bit] NOT NULL DEFAULT 0, - [Office Mac] [bit] NOT NULL DEFAULT 0, - [Office Mobile] [bit] NOT NULL DEFAULT 0, - [Office Web] [bit] NOT NULL DEFAULT 0, - [Office Outlook] [bit] NOT NULL DEFAULT 0, - [Office Word] [bit] NOT NULL DEFAULT 0, - [Office Excel] [bit] NOT NULL DEFAULT 0, - [Office Powerpoint] [bit] NOT NULL DEFAULT 0, - [Office Onenote] [bit] NOT NULL DEFAULT 0, - [Office Teams] [bit] NOT NULL DEFAULT 0, - [Office Outlook Windows] [bit] NOT NULL DEFAULT 0, - [Office Word Windows] [bit] NOT NULL DEFAULT 0, - [Office Excel Windows] [bit] NOT NULL DEFAULT 0, - [Office Powerpoint Windows] [bit] NOT NULL DEFAULT 0, - [Office Onenote Windows] [bit] NOT NULL DEFAULT 0, - [Office Teams Windows] [bit] NOT NULL DEFAULT 0, - [Office Outlook Mac] [bit] NOT NULL DEFAULT 0, - [Office Word Mac] [bit] NOT NULL DEFAULT 0, - [Office Excel Mac] [bit] NOT NULL DEFAULT 0, - [Office Powerpoint Mac] [bit] NOT NULL DEFAULT 0, - [Office Onenote Mac] [bit] NOT NULL DEFAULT 0, - [Office Teams Mac] [bit] NOT NULL DEFAULT 0, - [Office Outlook Mobile] [bit] NOT NULL DEFAULT 0, - [Office Word Mobile] [bit] NOT NULL DEFAULT 0, - [Office Excel Mobile] [bit] NOT NULL DEFAULT 0, - [Office Powerpoint Mobile] [bit] NOT NULL DEFAULT 0, - [Office Onenote Mobile] [bit] NOT NULL DEFAULT 0, - [Office Teams Mobile] [bit] NOT NULL DEFAULT 0, - [Office Outlook Web] [bit] NOT NULL DEFAULT 0, - [Office Word Web] [bit] NOT NULL DEFAULT 0, - [Office Excel Web] [bit] NOT NULL DEFAULT 0, - [Office Powerpoint Web] [bit] NOT NULL DEFAULT 0, - [Office Onenote Web] [bit] NOT NULL DEFAULT 0, - [Office Teams Web] [bit] NOT NULL DEFAULT 0, - -- Yammer devices - [Yammer Platform Count] [tinyint] NOT NULL DEFAULT 0, - [Yammer Used Web] [bit] NOT NULL DEFAULT 0, - [Yammer Used Mobile] [bit] NOT NULL DEFAULT 0, - [Yammer Used Others] [bit] NOT NULL DEFAULT 0, - [Yammer Used WinPhone] [bit] NOT NULL DEFAULT 0, - [Yammer Used Android] [bit] NOT NULL DEFAULT 0, - [Yammer Used iPad] [bit] NOT NULL DEFAULT 0, - [Yammer Used iPhone] [bit] NOT NULL DEFAULT 0, - CONSTRAINT [PK_profiling.UsageWeekly] PRIMARY KEY CLUSTERED ([user_id], [date] ASC) WITH ( - STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF - ) ON [PRIMARY] +BEGIN + CREATE TABLE profiling.UsageWeekly + ( + "user_id" INT NOT NULL, + "date" DATE NOT NULL, + "Teams Used Web" BIT NOT NULL DEFAULT 0, + "Teams Used Mac" BIT NOT NULL DEFAULT 0, + "Teams Used Windows" BIT NOT NULL DEFAULT 0, + "Teams Used Linux" BIT NOT NULL DEFAULT 0, + "Teams Used Chrome OS" BIT NOT NULL DEFAULT 0, + "Teams Used Mobile" BIT NOT NULL DEFAULT 0, + "Teams Used WinPhone" BIT NOT NULL DEFAULT 0, + "Teams Used iOS" BIT NOT NULL DEFAULT 0, + "Teams Used Android" BIT NOT NULL DEFAULT 0, + "Office Windows" BIT NOT NULL DEFAULT 0, + "Office Mac" BIT NOT NULL DEFAULT 0, + "Office Mobile" BIT NOT NULL DEFAULT 0, + "Office Web" BIT NOT NULL DEFAULT 0, + "Office Outlook" BIT NOT NULL DEFAULT 0, + "Office Word" BIT NOT NULL DEFAULT 0, + "Office Excel" BIT NOT NULL DEFAULT 0, + "Office Powerpoint" BIT NOT NULL DEFAULT 0, + "Office Onenote" BIT NOT NULL DEFAULT 0, + "Office Teams" BIT NOT NULL DEFAULT 0, + "Office Outlook Windows" BIT NOT NULL DEFAULT 0, + "Office Word Windows" BIT NOT NULL DEFAULT 0, + "Office Excel Windows" BIT NOT NULL DEFAULT 0, + "Office Powerpoint Windows" BIT NOT NULL DEFAULT 0, + "Office Onenote Windows" BIT NOT NULL DEFAULT 0, + "Office Teams Windows" BIT NOT NULL DEFAULT 0, + "Office Outlook Mac" BIT NOT NULL DEFAULT 0, + "Office Word Mac" BIT NOT NULL DEFAULT 0, + "Office Excel Mac" BIT NOT NULL DEFAULT 0, + "Office Powerpoint Mac" BIT NOT NULL DEFAULT 0, + "Office Onenote Mac" BIT NOT NULL DEFAULT 0, + "Office Teams Mac" BIT NOT NULL DEFAULT 0, + "Office Outlook Mobile" BIT NOT NULL DEFAULT 0, + "Office Word Mobile" BIT NOT NULL DEFAULT 0, + "Office Excel Mobile" BIT NOT NULL DEFAULT 0, + "Office Powerpoint Mobile" BIT NOT NULL DEFAULT 0, + "Office Onenote Mobile" BIT NOT NULL DEFAULT 0, + "Office Teams Mobile" BIT NOT NULL DEFAULT 0, + "Office Outlook Web" BIT NOT NULL DEFAULT 0, + "Office Word Web" BIT NOT NULL DEFAULT 0, + "Office Excel Web" BIT NOT NULL DEFAULT 0, + "Office Powerpoint Web" BIT NOT NULL DEFAULT 0, + "Office Onenote Web" BIT NOT NULL DEFAULT 0, + "Office Teams Web" BIT NOT NULL DEFAULT 0, + "Yammer Platform Count" TINYINT NOT NULL DEFAULT 0, + "Yammer Used Web" BIT NOT NULL DEFAULT 0, + "Yammer Used Mobile" BIT NOT NULL DEFAULT 0, + "Yammer Used Others" BIT NOT NULL DEFAULT 0, + "Yammer Used WinPhone" BIT NOT NULL DEFAULT 0, + "Yammer Used Android" BIT NOT NULL DEFAULT 0, + "Yammer Used iPad" BIT NOT NULL DEFAULT 0, + "Yammer Used iPhone" BIT NOT NULL DEFAULT 0, + CONSTRAINT [PK_profiling.UsageWeekly] + PRIMARY KEY CLUSTERED ("user_id", "date" ASC) + WITH ( + STATISTICS_NORECOMPUTE = OFF + ,IGNORE_DUP_KEY = OFF + -- ,OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF + ) ); +END GO -- ========================= @@ -369,24 +718,31 @@ GO -- =========================================== -- From a date, return the Monday of that week -- =========================================== + IF OBJECT_ID(N'profiling.udf_GetMonday') IS NOT NULL - DROP FUNCTION [profiling].[udf_GetMonday]; +BEGIN + DROP FUNCTION profiling.udf_GetMonday; +END GO -CREATE FUNCTION [profiling].[udf_GetMonday] (@CurrentDate DATE) +CREATE FUNCTION profiling.udf_GetMonday (@CurrentDate DATE) RETURNS DATE AS BEGIN - DECLARE @Result DATE, @ThisDayWasMonday INT; + DECLARE + @Result DATE, + @ThisDayWasMonday INT; + -- Set the first day of the week to Monday - SET @ThisDayWasMonday = DATEPART(WEEKDAY, '20230102'); - SET @RESULT = @CurrentDate; + SELECT + @ThisDayWasMonday = DATEPART(WEEKDAY, '20230102'), + @Result = @CurrentDate; WHILE @ThisDayWasMonday <> DATEPART(WEEKDAY, @Result) BEGIN - SET @Result = CONVERT(DATE, DATEADD(DAY, -1, @Result)); + SELECT @Result = CONVERT(DATE, DATEADD(DAY, -1, @Result)); END RETURN @Result; -END +END; GO -- ============================================================ @@ -397,954 +753,2021 @@ GO -- ============================================================ -- First drop the SPs, then the Table Types -IF OBJECT_ID(N'profiling.usp_UpsertTeams') IS NOT NULL DROP PROCEDURE [profiling].[usp_UpsertTeams]; +IF OBJECT_ID(N'profiling.usp_UpsertTeams') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertTeams; +END +GO + +IF OBJECT_ID(N'profiling.usp_UpsertOneDrive') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertOneDrive; +END +GO + +IF OBJECT_ID(N'profiling.usp_UpsertSharePoint') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertSharePoint; +END GO -IF OBJECT_ID(N'profiling.usp_UpsertOneDrive') IS NOT NULL DROP PROCEDURE [profiling].[usp_UpsertOneDrive]; + +IF OBJECT_ID(N'profiling.usp_UpsertOutlook') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertOutlook; +END GO -IF OBJECT_ID(N'profiling.usp_UpsertSharePoint') IS NOT NULL DROP PROCEDURE [profiling].[usp_UpsertSharePoint]; + +IF OBJECT_ID(N'profiling.usp_UpsertYammer') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertYammer; +END GO -IF OBJECT_ID(N'profiling.usp_UpsertOutlook') IS NOT NULL DROP PROCEDURE [profiling].[usp_UpsertOutlook]; + +IF OBJECT_ID(N'profiling.usp_UpsertTeamsDevices') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertTeamsDevices; +END GO -IF OBJECT_ID(N'profiling.usp_UpsertYammer') IS NOT NULL DROP PROCEDURE [profiling].[usp_UpsertYammer]; + +IF OBJECT_ID(N'profiling.usp_UpsertM365Apps') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertM365Apps; +END GO -IF OBJECT_ID(N'profiling.usp_UpsertTeamsDevices') IS NOT NULL DROP PROCEDURE [profiling].[usp_UpsertTeamsDevices]; + +IF OBJECT_ID(N'profiling.usp_UpsertYammerDevices') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertYammerDevices; +END GO -IF OBJECT_ID(N'profiling.usp_UpsertM365Apps') IS NOT NULL DROP PROCEDURE [profiling].[usp_UpsertM365Apps]; + +IF OBJECT_ID(N'profiling.usp_UpsertCopilot') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_UpsertCopilot; +END GO -IF OBJECT_ID(N'profiling.usp_UpsertYammerDevices') IS NOT NULL DROP PROCEDURE [profiling].[usp_UpsertYammerDevices]; + +IF TYPE_ID(N'ut_teams_user_activity_log') IS NOT NULL +BEGIN + DROP TYPE ut_teams_user_activity_log; +END GO -IF TYPE_ID(N'ut_teams_user_activity_log') IS NOT NULL DROP TYPE ut_teams_user_activity_log; + +IF TYPE_ID(N'ut_onedrive_user_activity_log') IS NOT NULL +BEGIN + DROP TYPE ut_onedrive_user_activity_log; +END GO -IF TYPE_ID(N'ut_onedrive_user_activity_log') IS NOT NULL DROP TYPE ut_onedrive_user_activity_log; + +IF TYPE_ID(N'ut_sharepoint_user_activity_log') IS NOT NULL +BEGIN + DROP TYPE ut_sharepoint_user_activity_log; +END GO -IF TYPE_ID(N'ut_sharepoint_user_activity_log') IS NOT NULL DROP TYPE ut_sharepoint_user_activity_log; + +IF TYPE_ID(N'ut_outlook_user_activity_log') IS NOT NULL +BEGIN + DROP TYPE ut_outlook_user_activity_log; +END GO -IF TYPE_ID(N'ut_outlook_user_activity_log') IS NOT NULL DROP TYPE ut_outlook_user_activity_log; + +IF TYPE_ID(N'ut_yammer_user_activity_log') IS NOT NULL +BEGIN + DROP TYPE ut_yammer_user_activity_log; +END GO -IF TYPE_ID(N'ut_yammer_user_activity_log') IS NOT NULL DROP TYPE ut_yammer_user_activity_log; + +IF TYPE_ID(N'ut_teams_user_device_usage_log') IS NOT NULL +BEGIN + DROP TYPE ut_teams_user_device_usage_log; +END GO -IF TYPE_ID(N'ut_teams_user_device_usage_log') IS NOT NULL DROP TYPE ut_teams_user_device_usage_log; + +IF TYPE_ID(N'ut_platform_user_activity_log') IS NOT NULL +BEGIN + DROP TYPE ut_platform_user_activity_log; +END GO -IF TYPE_ID(N'ut_platform_user_activity_log') IS NOT NULL DROP TYPE ut_platform_user_activity_log; + +IF TYPE_ID(N'ut_yammer_device_activity_log') IS NOT NULL +BEGIN + DROP TYPE ut_yammer_device_activity_log; +END GO -IF TYPE_ID(N'ut_yammer_device_activity_log') IS NOT NULL DROP TYPE ut_yammer_device_activity_log; + +IF TYPE_ID(N'ut_copilot_activities') IS NOT NULL +BEGIN + DROP TYPE ut_copilot_activities; +END GO + -- Add Table Types -IF TYPE_ID(N'ut_teams_user_activity_log') IS NULL CREATE TYPE ut_teams_user_activity_log AS TABLE ( - [user_id] INT NOT NULL, - [date] DATETIME NOT NULL, - [private_chat_count] BIGINT NOT NULL, - [team_chat_count] BIGINT NOT NULL, - [calls_count] BIGINT NOT NULL, - [meetings_count] BIGINT NOT NULL, - [meetings_attended_count] BIGINT NOT NULL, - [meetings_organized_count] BIGINT NOT NULL, - [adhoc_meetings_attended_count] BIGINT NOT NULL, - [adhoc_meetings_organized_count] BIGINT NOT NULL, - [scheduled_onetime_meetings_attended_count] BIGINT NOT NULL, - [scheduled_onetime_meetings_organized_count] BIGINT NOT NULL, - [scheduled_recurring_meetings_attended_count] BIGINT NOT NULL, - [scheduled_recurring_meetings_organized_count] BIGINT NOT NULL, - [audio_duration_seconds] INT NOT NULL, - [video_duration_seconds] INT NOT NULL, - [screenshare_duration_seconds] INT NOT NULL, - [post_messages] BIGINT NOT NULL, - [reply_messages] BIGINT NOT NULL, - [urgent_messages] BIGINT NOT NULL -); -GO -IF TYPE_ID(N'ut_onedrive_user_activity_log') IS NULL CREATE TYPE ut_onedrive_user_activity_log AS TABLE ( - [user_id] INT NOT NULL, - [date] DATETIME NOT NULL, - [viewed_or_edited] BIGINT NOT NULL, - [synced] BIGINT NOT NULL, - [shared_internally] BIGINT NOT NULL, - [shared_externally] BIGINT NOT NULL -); -GO -IF TYPE_ID(N'ut_sharepoint_user_activity_log') IS NULL CREATE TYPE ut_sharepoint_user_activity_log AS TABLE ( - [user_id] INT NOT NULL, - [date] DATETIME NOT NULL, - [viewed_or_edited] BIGINT NOT NULL, - [synced] BIGINT NOT NULL, - [shared_internally] BIGINT NOT NULL, - [shared_externally] BIGINT NOT NULL -); -GO -IF TYPE_ID(N'ut_outlook_user_activity_log') IS NULL CREATE TYPE ut_outlook_user_activity_log AS TABLE ( - [user_id] INT NOT NULL, - [date] DATETIME NOT NULL, - [email_send_count] BIGINT NOT NULL, - [email_receive_count] BIGINT NOT NULL, - [email_read_count] BIGINT NOT NULL, - [meeting_created_count] BIGINT NOT NULL, - [meeting_interacted_count] BIGINT NOT NULL -); -GO -IF TYPE_ID(N'ut_yammer_user_activity_log') IS NULL CREATE TYPE ut_yammer_user_activity_log AS TABLE ( - [user_id] INT NOT NULL, - [date] DATETIME NOT NULL, - [posted_count] INT NOT NULL, - [read_count] INT NOT NULL, - [liked_count] INT NOT NULL -); -GO -IF TYPE_ID(N'ut_teams_user_device_usage_log') IS NULL CREATE TYPE ut_teams_user_device_usage_log AS TABLE ( - [user_id] INT NOT NULL, - [date] DATETIME NOT NULL, - [used_web] [int] NOT NULL DEFAULT 0, - [used_mac] [int] NOT NULL DEFAULT 0, - [used_windows] [int] NOT NULL DEFAULT 0, - [used_linux] [int] NOT NULL DEFAULT 0, - [used_chrome_os] [int] NOT NULL DEFAULT 0, - [used_mobile] [int] NOT NULL DEFAULT 0, - [used_win_phone] [int] NOT NULL DEFAULT 0, - [used_ios] [int] NOT NULL DEFAULT 0, - [used_android] [int] NOT NULL DEFAULT 0 -); -GO -IF TYPE_ID(N'ut_platform_user_activity_log') IS NULL CREATE TYPE ut_platform_user_activity_log AS TABLE ( - [user_id] INT NOT NULL, - [date] DATETIME NOT NULL, - [windows] [int] NOT NULL DEFAULT 0, - [mac] [int] NOT NULL DEFAULT 0, - [mobile] [int] NOT NULL DEFAULT 0, - [web] [int] NOT NULL DEFAULT 0, - [outlook] [int] NOT NULL DEFAULT 0, - [word] [int] NOT NULL DEFAULT 0, - [excel] [int] NOT NULL DEFAULT 0, - [powerpoint] [int] NOT NULL DEFAULT 0, - [onenote] [int] NOT NULL DEFAULT 0, - [teams] [int] NOT NULL DEFAULT 0, - [outlook_windows] [int] NOT NULL DEFAULT 0, - [word_windows] [int] NOT NULL DEFAULT 0, - [excel_windows] [int] NOT NULL DEFAULT 0, - [powerpoint_windows] [int] NOT NULL DEFAULT 0, - [onenote_windows] [int] NOT NULL DEFAULT 0, - [teams_windows] [int] NOT NULL DEFAULT 0, - [outlook_mac] [int] NOT NULL DEFAULT 0, - [word_mac] [int] NOT NULL DEFAULT 0, - [excel_mac] [int] NOT NULL DEFAULT 0, - [powerpoint_mac] [int] NOT NULL DEFAULT 0, - [onenote_mac] [int] NOT NULL DEFAULT 0, - [teams_mac] [int] NOT NULL DEFAULT 0, - [outlook_mobile] [int] NOT NULL DEFAULT 0, - [word_mobile] [int] NOT NULL DEFAULT 0, - [excel_mobile] [int] NOT NULL DEFAULT 0, - [powerpoint_mobile] [int] NOT NULL DEFAULT 0, - [onenote_mobile] [int] NOT NULL DEFAULT 0, - [teams_mobile] [int] NOT NULL DEFAULT 0, - [outlook_web] [int] NOT NULL DEFAULT 0, - [word_web] [int] NOT NULL DEFAULT 0, - [excel_web] [int] NOT NULL DEFAULT 0, - [powerpoint_web] [int] NOT NULL DEFAULT 0, - [onenote_web] [int] NOT NULL DEFAULT 0, - [teams_web] [int] NOT NULL DEFAULT 0 -); -GO -IF TYPE_ID(N'ut_yammer_device_activity_log') IS NULL CREATE TYPE ut_yammer_device_activity_log AS TABLE ( - [user_id] INT NOT NULL, - [date] DATETIME NOT NULL, - [used_count] [int] NOT NULL, - [used_web] [int] NOT NULL, - [used_mobile] [int] NOT NULL, - [used_others] [int] NOT NULL, - [used_win_phone] [int] NOT NULL, - [used_android] [int] NOT NULL, - [used_ipad] [int] NOT NULL, - [used_iphone] [int] NOT NULL -); + +IF TYPE_ID(N'ut_teams_user_activity_log') IS NULL +BEGIN + CREATE TYPE ut_teams_user_activity_log AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + private_chat_count BIGINT NOT NULL, + team_chat_count BIGINT NOT NULL, + calls_count BIGINT NOT NULL, + meetings_count BIGINT NOT NULL, + meetings_attended_count BIGINT NOT NULL, + meetings_organized_count BIGINT NOT NULL, + adhoc_meetings_attended_count BIGINT NOT NULL, + adhoc_meetings_organized_count BIGINT NOT NULL, + scheduled_onetime_meetings_attended_count BIGINT NOT NULL, + scheduled_onetime_meetings_organized_count BIGINT NOT NULL, + scheduled_recurring_meetings_attended_count BIGINT NOT NULL, + scheduled_recurring_meetings_organized_count BIGINT NOT NULL, + audio_duration_seconds INT NOT NULL, + video_duration_seconds INT NOT NULL, + screenshare_duration_seconds INT NOT NULL, + post_messages BIGINT NOT NULL, + reply_messages BIGINT NOT NULL, + urgent_messages BIGINT NOT NULL + ); +END +GO + +IF TYPE_ID(N'ut_onedrive_user_activity_log') IS NULL +BEGIN + CREATE TYPE ut_onedrive_user_activity_log AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + viewed_or_edited BIGINT NOT NULL, + synced BIGINT NOT NULL, + shared_internally BIGINT NOT NULL, + shared_externally BIGINT NOT NULL + ); +END +GO + +IF TYPE_ID(N'ut_sharepoint_user_activity_log') IS NULL +BEGIN + CREATE TYPE ut_sharepoint_user_activity_log AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + viewed_or_edited BIGINT NOT NULL, + synced BIGINT NOT NULL, + shared_internally BIGINT NOT NULL, + shared_externally BIGINT NOT NULL + ); +END +GO + +IF TYPE_ID(N'ut_outlook_user_activity_log') IS NULL +BEGIN + CREATE TYPE ut_outlook_user_activity_log AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + email_send_count BIGINT NOT NULL, + email_receive_count BIGINT NOT NULL, + email_read_count BIGINT NOT NULL, + meeting_created_count BIGINT NOT NULL, + meeting_interacted_count BIGINT NOT NULL + ); +END +GO + +IF TYPE_ID(N'ut_yammer_user_activity_log') IS NULL +BEGIN + CREATE TYPE ut_yammer_user_activity_log AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + posted_count INT NOT NULL, + read_count INT NOT NULL, + liked_count INT NOT NULL + ); +END +GO + +IF TYPE_ID(N'ut_teams_user_device_usage_log') IS NULL +BEGIN + CREATE TYPE ut_teams_user_device_usage_log AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + used_web INT NOT NULL DEFAULT 0, + used_mac INT NOT NULL DEFAULT 0, + used_windows INT NOT NULL DEFAULT 0, + used_linux INT NOT NULL DEFAULT 0, + used_chrome_os INT NOT NULL DEFAULT 0, + used_mobile INT NOT NULL DEFAULT 0, + used_win_phone INT NOT NULL DEFAULT 0, + used_ios INT NOT NULL DEFAULT 0, + used_android INT NOT NULL DEFAULT 0 + ); +END +GO + +IF TYPE_ID(N'ut_platform_user_activity_log') IS NULL +BEGIN + CREATE TYPE ut_platform_user_activity_log AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + windows INT NOT NULL DEFAULT 0, + mac INT NOT NULL DEFAULT 0, + mobile INT NOT NULL DEFAULT 0, + web INT NOT NULL DEFAULT 0, + outlook INT NOT NULL DEFAULT 0, + word INT NOT NULL DEFAULT 0, + excel INT NOT NULL DEFAULT 0, + powerpoint INT NOT NULL DEFAULT 0, + onenote INT NOT NULL DEFAULT 0, + teams INT NOT NULL DEFAULT 0, + outlook_windows INT NOT NULL DEFAULT 0, + word_windows INT NOT NULL DEFAULT 0, + excel_windows INT NOT NULL DEFAULT 0, + powerpoint_windows INT NOT NULL DEFAULT 0, + onenote_windows INT NOT NULL DEFAULT 0, + teams_windows INT NOT NULL DEFAULT 0, + outlook_mac INT NOT NULL DEFAULT 0, + word_mac INT NOT NULL DEFAULT 0, + excel_mac INT NOT NULL DEFAULT 0, + powerpoint_mac INT NOT NULL DEFAULT 0, + onenote_mac INT NOT NULL DEFAULT 0, + teams_mac INT NOT NULL DEFAULT 0, + outlook_mobile INT NOT NULL DEFAULT 0, + word_mobile INT NOT NULL DEFAULT 0, + excel_mobile INT NOT NULL DEFAULT 0, + powerpoint_mobile INT NOT NULL DEFAULT 0, + onenote_mobile INT NOT NULL DEFAULT 0, + teams_mobile INT NOT NULL DEFAULT 0, + outlook_web INT NOT NULL DEFAULT 0, + word_web INT NOT NULL DEFAULT 0, + excel_web INT NOT NULL DEFAULT 0, + powerpoint_web INT NOT NULL DEFAULT 0, + onenote_web INT NOT NULL DEFAULT 0, + teams_web INT NOT NULL DEFAULT 0 + ); +END +GO + +IF TYPE_ID(N'ut_yammer_device_activity_log') IS NULL +BEGIN + CREATE TYPE ut_yammer_device_activity_log AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + used_count INT NOT NULL, + used_web INT NOT NULL, + used_mobile INT NOT NULL, + used_others INT NOT NULL, + used_win_phone INT NOT NULL, + used_android INT NOT NULL, + used_ipad INT NOT NULL, + used_iphone INT NOT NULL + ); +END +GO + +IF TYPE_ID(N'ut_copilot_activities') IS NULL +BEGIN + CREATE TYPE ut_copilot_activities AS TABLE ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + copilot_chats BIGINT NOT NULL DEFAULT 0, + copilot_meetings BIGINT NOT NULL DEFAULT 0, + copilot_files BIGINT NOT NULL DEFAULT 0, + copilot_assist365 BIGINT NOT NULL DEFAULT 0, + copilot_bing BIGINT NOT NULL DEFAULT 0, + copilot_bashtool BIGINT NOT NULL DEFAULT 0, + copilot_devui BIGINT NOT NULL DEFAULT 0, + copilot_excel BIGINT NOT NULL DEFAULT 0, + copilot_loop BIGINT NOT NULL DEFAULT 0, + copilot_m365admincenter BIGINT NOT NULL DEFAULT 0, + copilot_m365app BIGINT NOT NULL DEFAULT 0, + copilot_office BIGINT NOT NULL DEFAULT 0, + copilot_onenote BIGINT NOT NULL DEFAULT 0, + copilot_outlook BIGINT NOT NULL DEFAULT 0, + copilot_planner BIGINT NOT NULL DEFAULT 0, + copilot_powerpoint BIGINT NOT NULL DEFAULT 0, + copilot_sharepoint BIGINT NOT NULL DEFAULT 0, + copilot_stream BIGINT NOT NULL DEFAULT 0, + copilot_teams BIGINT NOT NULL DEFAULT 0, + copilot_vivacopilot BIGINT NOT NULL DEFAULT 0, + copilot_vivaengage BIGINT NOT NULL DEFAULT 0, + copilot_vivagoals BIGINT NOT NULL DEFAULT 0, + copilot_whiteboard BIGINT NOT NULL DEFAULT 0, + copilot_word BIGINT NOT NULL DEFAULT 0 + ); +END GO -- Add SPs -CREATE PROCEDURE [profiling].[usp_UpsertTeams] ( - @StartDate DATE, @EndDate DATE -) AS + +/* +Upsert procedures should get the raw data from the database and then process it +and push it to the staging temp table. +There are two staging tables, depending on the type of data: +- Activity count data (user sent X emails that day) goes into #ActivitiesStaging +- Boolean data (Teams mobile was used or not that day) goes into #UsageStaging +*/ +CREATE PROCEDURE profiling.usp_UpsertTeams +( + @StartDate DATE, + @EndDate DATE +) +AS BEGIN SET NOCOUNT ON; DECLARE @teams AS ut_teams_user_activity_log; - - INSERT INTO @teams ( - [user_id], [date], - [private_chat_count], [team_chat_count], [calls_count], - [meetings_count], [meetings_attended_count], [meetings_organized_count], - [adhoc_meetings_attended_count], [adhoc_meetings_organized_count], - [scheduled_onetime_meetings_attended_count], [scheduled_onetime_meetings_organized_count], - [scheduled_recurring_meetings_attended_count], [scheduled_recurring_meetings_organized_count], - [audio_duration_seconds], [video_duration_seconds], [screenshare_duration_seconds], - [post_messages], [reply_messages], [urgent_messages] + INSERT INTO @teams + ( + "user_id", + "date", + private_chat_count, + team_chat_count, + calls_count, + meetings_count, + meetings_attended_count, + meetings_organized_count, + adhoc_meetings_attended_count, + adhoc_meetings_organized_count, + scheduled_onetime_meetings_attended_count, + scheduled_onetime_meetings_organized_count, + scheduled_recurring_meetings_attended_count, + scheduled_recurring_meetings_organized_count, + audio_duration_seconds, + video_duration_seconds, + screenshare_duration_seconds, + post_messages, + reply_messages, + urgent_messages ) SELECT - [user_id], @StartDate, - SUM([private_chat_count]), SUM([team_chat_count]), SUM([calls_count]), - SUM([meetings_count]), SUM([meetings_attended_count]), SUM([meetings_organized_count]), - SUM([adhoc_meetings_attended_count]), SUM([adhoc_meetings_organized_count]), - SUM([scheduled_onetime_meetings_attended_count]), SUM([scheduled_onetime_meetings_organized_count]), - SUM([scheduled_recurring_meetings_attended_count]), SUM([scheduled_recurring_meetings_organized_count]), - SUM([audio_duration_seconds]), SUM([video_duration_seconds]), SUM([screenshare_duration_seconds]), - SUM([post_messages]), SUM([reply_messages]), SUM([urgent_messages]) - FROM dbo.[teams_user_activity_log] - WHERE @StartDate <= [date] AND [date] <= @EndDate - GROUP BY [user_id]; - + "user_id", + @StartDate, + SUM(private_chat_count), + SUM(team_chat_count), + SUM(calls_count), + SUM(meetings_count), + SUM(meetings_attended_count), + SUM(meetings_organized_count), + SUM(adhoc_meetings_attended_count), + SUM(adhoc_meetings_organized_count), + SUM(scheduled_onetime_meetings_attended_count), + SUM(scheduled_onetime_meetings_organized_count), + SUM(scheduled_recurring_meetings_attended_count), + SUM(scheduled_recurring_meetings_organized_count), + SUM(audio_duration_seconds), + SUM(video_duration_seconds), + SUM(screenshare_duration_seconds), + SUM(post_messages), + SUM(reply_messages), + SUM(urgent_messages) + FROM dbo.teams_user_activity_log + WHERE @StartDate <= "date" AND "date" <= @EndDate + GROUP BY "user_id"; + + /* tsqllint-disable warning update-where */ UPDATE t WITH (UPDLOCK, SERIALIZABLE) SET - [Teams Private Chats] = tvp.[private_chat_count], - [Teams Team Chats] = tvp.[team_chat_count], - [Teams Calls] = tvp.[calls_count], - [Teams Meetings] = tvp.[meetings_count], - [Teams Meetings Attended] = tvp.[meetings_attended_count], - [Teams Meetings Organized] = tvp.[meetings_organized_count], - [Teams Adhoc Meetings Attended] = tvp.[adhoc_meetings_attended_count], - [Teams Adhoc Meetings Organized] = tvp.[adhoc_meetings_organized_count], - [Teams Scheduled Onetime Meetings Attended] = tvp.[scheduled_onetime_meetings_attended_count], - [Teams Scheduled Onetime Meetings Organized] = tvp.[scheduled_onetime_meetings_organized_count], - [Teams Scheduled Recurring Meetings Attended] = tvp.[scheduled_recurring_meetings_attended_count], - [Teams Scheduled Recurring Meetings Organized] = tvp.[scheduled_recurring_meetings_organized_count], - [Teams Audio Duration Seconds] = tvp.[audio_duration_seconds], - [Teams Video Duration Seconds] = tvp.[video_duration_seconds], - [Teams Screenshare Duration Seconds] = tvp.[screenshare_duration_seconds], - [Teams Post Messages] = tvp.[post_messages], - [Teams Reply Messages] = tvp.[reply_messages], - [Teams Urgent Messages] = tvp.[urgent_messages] + "Teams Private Chats" = tvp.private_chat_count, + "Teams Team Chats" = tvp.team_chat_count, + "Teams Calls" = tvp.calls_count, + "Teams Meetings" = tvp.meetings_count, + "Teams Meetings Attended" = tvp.meetings_attended_count, + "Teams Meetings Organized" = tvp.meetings_organized_count, + "Teams Adhoc Meetings Attended" = tvp.adhoc_meetings_attended_count, + "Teams Adhoc Meetings Organized" = tvp.adhoc_meetings_organized_count, + "Teams Scheduled Onetime Meetings Attended" = tvp.scheduled_onetime_meetings_attended_count, + "Teams Scheduled Onetime Meetings Organized" = tvp.scheduled_onetime_meetings_organized_count, + "Teams Scheduled Recurring Meetings Attended" = tvp.scheduled_recurring_meetings_attended_count, + "Teams Scheduled Recurring Meetings Organized" = tvp.scheduled_recurring_meetings_organized_count, + "Teams Audio Duration Seconds" = tvp.audio_duration_seconds, + "Teams Video Duration Seconds" = tvp.video_duration_seconds, + "Teams Screenshare Duration Seconds" = tvp.screenshare_duration_seconds, + "Teams Post Messages" = tvp.post_messages, + "Teams Reply Messages" = tvp.reply_messages, + "Teams Urgent Messages" = tvp.urgent_messages FROM #ActivitiesStaging AS t - INNER JOIN @teams AS tvp ON t.user_id = tvp.user_id AND t.date = tvp.date; - - INSERT #ActivitiesStaging ( - user_id, date, - [Teams Private Chats], [Teams Team Chats], [Teams Calls], - [Teams Meetings], [Teams Meetings Attended], [Teams Meetings Organized], - [Teams Adhoc Meetings Attended], [Teams Adhoc Meetings Organized], - [Teams Scheduled Onetime Meetings Attended], [Teams Scheduled Onetime Meetings Organized], - [Teams Scheduled Recurring Meetings Attended], [Teams Scheduled Recurring Meetings Organized], - [Teams Audio Duration Seconds], [Teams Video Duration Seconds], [Teams Screenshare Duration Seconds], - [Teams Post Messages], [Teams Reply Messages], [Teams Urgent Messages] + INNER JOIN @teams AS tvp + ON t."user_id" = tvp."user_id" AND t."date" = tvp."date"; + /* tsqllint-enable warning update-where */ + + INSERT #ActivitiesStaging + ( + "user_id", + "date", + "Teams Private Chats", + "Teams Team Chats", + "Teams Calls", + "Teams Meetings", + "Teams Meetings Attended", + "Teams Meetings Organized", + "Teams Adhoc Meetings Attended", + "Teams Adhoc Meetings Organized", + "Teams Scheduled Onetime Meetings Attended", + "Teams Scheduled Onetime Meetings Organized", + "Teams Scheduled Recurring Meetings Attended", + "Teams Scheduled Recurring Meetings Organized", + "Teams Audio Duration Seconds", + "Teams Video Duration Seconds", + "Teams Screenshare Duration Seconds", + "Teams Post Messages", + "Teams Reply Messages", + "Teams Urgent Messages" ) SELECT - user_id, date, - [private_chat_count], [team_chat_count], [calls_count], - [meetings_count], [meetings_attended_count], [meetings_organized_count], - [adhoc_meetings_attended_count], [adhoc_meetings_organized_count], - [scheduled_onetime_meetings_attended_count], [scheduled_onetime_meetings_organized_count], - [scheduled_recurring_meetings_attended_count], [scheduled_recurring_meetings_organized_count], - [audio_duration_seconds], [video_duration_seconds], [screenshare_duration_seconds], - [post_messages], [reply_messages], [urgent_messages] - FROM @teams as tvp - WHERE NOT EXISTS (SELECT 1 FROM #ActivitiesStaging as t WHERE t.user_id = tvp.user_id AND t.date = tvp.date); -END + "user_id", + "date", + private_chat_count, + team_chat_count, + calls_count, + meetings_count, + meetings_attended_count, + meetings_organized_count, + adhoc_meetings_attended_count, + adhoc_meetings_organized_count, + scheduled_onetime_meetings_attended_count, + scheduled_onetime_meetings_organized_count, + scheduled_recurring_meetings_attended_count, + scheduled_recurring_meetings_organized_count, + audio_duration_seconds, + video_duration_seconds, + screenshare_duration_seconds, + post_messages, + reply_messages, + urgent_messages + FROM @teams AS tvp + WHERE NOT EXISTS ( + SELECT 1 + FROM #ActivitiesStaging AS t + WHERE t."user_id" = tvp."user_id" AND t."date" = tvp."date" + ); +END; GO -CREATE PROCEDURE [profiling].[usp_UpsertOneDrive] ( - @StartDate DATE, @EndDate DATE -) AS +CREATE PROCEDURE profiling.usp_UpsertOneDrive +( + @StartDate DATE, + @EndDate DATE +) +AS BEGIN SET NOCOUNT ON; DECLARE @onedrive AS ut_onedrive_user_activity_log; - - INSERT INTO @onedrive ( - [user_id], [date], - [viewed_or_edited], [synced], [shared_internally], [shared_externally] + INSERT INTO @onedrive + ( + "user_id", + "date", + viewed_or_edited, + synced, + shared_internally, + shared_externally ) SELECT - [user_id], @StartDate, - SUM([viewed_or_edited]), SUM([synced]), SUM([shared_internally]), SUM([shared_externally]) - FROM dbo.[onedrive_user_activity_log] - WHERE @StartDate <= [date] AND [date] <= @EndDate - GROUP BY [user_id]; - + "user_id", + @StartDate, + SUM(viewed_or_edited), + SUM(synced), + SUM(shared_internally), + SUM(shared_externally) + FROM dbo.onedrive_user_activity_log + WHERE @StartDate <= "date" AND "date" <= @EndDate + GROUP BY "user_id"; + + /* tsqllint-disable warning update-where */ UPDATE t WITH (UPDLOCK, SERIALIZABLE) SET - [OneDrive Viewed/Edited] = tvp.[viewed_or_edited], - [OneDrive Synced] = tvp.[synced], - [OneDrive Shared Internally] = tvp.[shared_internally], - [OneDrive Shared Externally] = tvp.[shared_externally] + "OneDrive Viewed/Edited" = tvp.viewed_or_edited, + "OneDrive Synced" = tvp.synced, + "OneDrive Shared Internally" = tvp.shared_internally, + "OneDrive Shared Externally" = tvp.shared_externally FROM #ActivitiesStaging AS t - INNER JOIN @onedrive AS tvp ON t.user_id = tvp.user_id AND t.date = tvp.date; - - INSERT #ActivitiesStaging ( - user_id, date, - [OneDrive Viewed/Edited], [OneDrive Synced], [OneDrive Shared Internally], [OneDrive Shared Externally] + INNER JOIN @onedrive AS tvp + ON t."user_id" = tvp."user_id" AND t."date" = tvp."date"; + /* tsqllint-enable warning update-where */ + + INSERT #ActivitiesStaging + ( + "user_id", + "date", + "OneDrive Viewed/Edited", + "OneDrive Synced", + "OneDrive Shared Internally", + "OneDrive Shared Externally" ) SELECT - user_id, date, - [viewed_or_edited], [synced], [shared_internally], [shared_externally] - FROM @onedrive as tvp - WHERE NOT EXISTS (SELECT 1 FROM #ActivitiesStaging as t WHERE t.user_id = tvp.user_id AND t.date = tvp.date); -END + "user_id", + "date", + viewed_or_edited, + synced, + shared_internally, + shared_externally + FROM @onedrive AS tvp + WHERE NOT EXISTS ( + SELECT 1 + FROM #ActivitiesStaging AS t + WHERE t."user_id" = tvp."user_id" AND t."date" = tvp."date" + ); +END; GO -CREATE PROCEDURE [profiling].[usp_UpsertSharePoint] ( - @StartDate DATE, @EndDate DATE -) AS +CREATE PROCEDURE profiling.usp_UpsertSharePoint +( + @StartDate DATE, + @EndDate DATE +) +AS BEGIN SET NOCOUNT ON; DECLARE @sharepoint AS ut_sharepoint_user_activity_log; - - INSERT INTO @sharepoint ( - [user_id], [date], - [viewed_or_edited], [synced], [shared_internally], [shared_externally] + INSERT INTO @sharepoint + ( + "user_id", + "date", + viewed_or_edited, + synced, + shared_internally, + shared_externally ) SELECT - [user_id], @StartDate, - SUM([viewed_or_edited]), SUM([synced]), SUM([shared_internally]), SUM([shared_externally]) - FROM dbo.[sharepoint_user_activity_log] - WHERE @StartDate <= [date] AND [date] <= @EndDate - GROUP BY [user_id]; - + "user_id", + @StartDate, + SUM(viewed_or_edited), + SUM(synced), + SUM(shared_internally), + SUM(shared_externally) + FROM dbo.sharepoint_user_activity_log + WHERE @StartDate <= "date" AND "date" <= @EndDate + GROUP BY "user_id"; + + /* tsqllint-disable warning update-where */ UPDATE t WITH (UPDLOCK, SERIALIZABLE) SET - [SPO Viewed/Edited] = tvp.[viewed_or_edited], - [SPO Synced] = tvp.[synced], - [SPO Shared Internally] = tvp.[shared_internally], - [SPO Shared Externally] = tvp.[shared_externally] + "SPO Viewed/Edited" = tvp.viewed_or_edited, + "SPO Synced" = tvp.synced, + "SPO Shared Internally" = tvp.shared_internally, + "SPO Shared Externally" = tvp.shared_externally FROM #ActivitiesStaging AS t - INNER JOIN @sharepoint AS tvp ON t.user_id = tvp.user_id AND t.date = tvp.date; - - INSERT #ActivitiesStaging ( - user_id, date, - [SPO Viewed/Edited], [SPO Synced], [SPO Shared Internally], [SPO Shared Externally] + INNER JOIN @sharepoint AS tvp + ON t."user_id" = tvp."user_id" AND t."date" = tvp."date"; + /* tsqllint-enable warning update-where */ + + INSERT #ActivitiesStaging + ( + "user_id", + "date", + "SPO Viewed/Edited", + "SPO Synced", + "SPO Shared Internally", + "SPO Shared Externally" ) SELECT - user_id, date, - [viewed_or_edited], [synced], [shared_internally], [shared_externally] - FROM @sharepoint as tvp - WHERE NOT EXISTS (SELECT 1 FROM #ActivitiesStaging as t WHERE t.user_id = tvp.user_id AND t.date = tvp.date); -END + "user_id", + "date", + viewed_or_edited, + synced, + shared_internally, + shared_externally + FROM @sharepoint AS tvp + WHERE NOT EXISTS ( + SELECT 1 + FROM #ActivitiesStaging AS t + WHERE t."user_id" = tvp."user_id" AND t."date" = tvp."date" + ); +END; GO -CREATE PROCEDURE [profiling].[usp_UpsertOutlook] ( - @StartDate DATE, @EndDate DATE -) AS +CREATE PROCEDURE profiling.usp_UpsertOutlook +( + @StartDate DATE, + @EndDate DATE +) +AS BEGIN SET NOCOUNT ON; DECLARE @outlook AS ut_outlook_user_activity_log; - - INSERT INTO @outlook ( - [user_id], [date], - [email_send_count], [email_receive_count], [email_read_count], - [meeting_created_count], [meeting_interacted_count] + INSERT INTO @outlook + ( + "user_id", + "date", + email_send_count, + email_receive_count, + email_read_count, + meeting_created_count, + meeting_interacted_count ) SELECT - [user_id], @StartDate, - SUM([email_send_count]), SUM([email_receive_count]), SUM([email_read_count]), - SUM([meeting_created_count]), SUM([meeting_interacted_count]) - FROM dbo.[outlook_user_activity_log] - WHERE @StartDate <= [date] AND [date] <= @EndDate - GROUP BY [user_id]; - + "user_id", + @StartDate, + SUM(email_send_count), + SUM(email_receive_count), + SUM(email_read_count), + SUM(meeting_created_count), + SUM(meeting_interacted_count) + FROM dbo.outlook_user_activity_log + WHERE @StartDate <= "date" AND "date" <= @EndDate + GROUP BY "user_id"; + + /* tsqllint-disable warning update-where */ UPDATE t WITH (UPDLOCK, SERIALIZABLE) SET - [Emails Sent] = tvp.[email_send_count], - [Emails Received] = tvp.[email_receive_count], - [Emails Read] = tvp.[email_read_count], - [Outlook Meetings Created] = tvp.[meeting_created_count], - [Outlook Meetings Interacted] = tvp.[meeting_interacted_count] + "Emails Sent" = tvp.email_send_count, + "Emails Received" = tvp.email_receive_count, + "Emails Read" = tvp.email_read_count, + "Outlook Meetings Created" = tvp.meeting_created_count, + "Outlook Meetings Interacted" = tvp.meeting_interacted_count FROM #ActivitiesStaging AS t - INNER JOIN @outlook AS tvp ON t.user_id = tvp.user_id AND t.date = tvp.date; - - INSERT #ActivitiesStaging ( - user_id, date, - [Emails Sent], [Emails Received], [Emails Read], - [Outlook Meetings Created], [Outlook Meetings Interacted] + INNER JOIN @outlook AS tvp + ON t."user_id" = tvp."user_id" AND t."date" = tvp."date"; + /* tsqllint-enable warning update-where */ + + INSERT #ActivitiesStaging + ( + "user_id", + "date", + "Emails Sent", + "Emails Received", + "Emails Read", + "Outlook Meetings Created", + "Outlook Meetings Interacted" ) SELECT - user_id, date, - [email_send_count], [email_receive_count], [email_read_count], - [meeting_created_count], [meeting_interacted_count] - FROM @outlook as tvp - WHERE NOT EXISTS (SELECT 1 FROM #ActivitiesStaging as t WHERE t.user_id = tvp.user_id AND t.date = tvp.date); -END + "user_id", + "date", + email_send_count, + email_receive_count, + email_read_count, + meeting_created_count, + meeting_interacted_count + FROM @outlook AS tvp + WHERE NOT EXISTS ( + SELECT 1 + FROM #ActivitiesStaging AS t + WHERE t."user_id" = tvp."user_id" AND t."date" = tvp."date" + ); +END; GO -CREATE PROCEDURE [profiling].[usp_UpsertYammer] ( - @StartDate DATE, @EndDate DATE -) AS +CREATE PROCEDURE profiling.usp_UpsertYammer +( + @StartDate DATE, + @EndDate DATE +) +AS BEGIN SET NOCOUNT ON; DECLARE @yammer AS ut_yammer_user_activity_log; - - INSERT INTO @yammer ( - [user_id], [date], - [posted_count], [read_count], [liked_count] + INSERT INTO @yammer + ( + "user_id", + "date", + posted_count, + read_count, + liked_count ) SELECT - [user_id], @StartDate, - SUM([posted_count]), SUM([read_count]), SUM([liked_count]) - FROM dbo.[yammer_user_activity_log] - WHERE @StartDate <= [date] AND [date] <= @EndDate - GROUP BY [user_id]; - + "user_id", + @StartDate, + SUM(posted_count), + SUM(read_count), + SUM(liked_count) + FROM dbo.yammer_user_activity_log + WHERE @StartDate <= "date" AND "date" <= @EndDate + GROUP BY "user_id"; + + /* tsqllint-disable warning update-where */ UPDATE t WITH (UPDLOCK, SERIALIZABLE) SET - [Yammer Posted] = tvp.[posted_count], - [Yammer Read] = tvp.[read_count], - [Yammer Liked] = tvp.[liked_count] + "Yammer Posted" = tvp.posted_count, + "Yammer Read" = tvp.read_count, + "Yammer Liked" = tvp.liked_count FROM #ActivitiesStaging AS t - INNER JOIN @yammer AS tvp ON t.user_id = tvp.user_id AND t.date = tvp.date; - - INSERT #ActivitiesStaging ( - user_id, date, - [Yammer Posted], [Yammer Read], [Yammer Liked] + INNER JOIN @yammer AS tvp + ON t."user_id" = tvp."user_id" AND t."date" = tvp."date"; + /* tsqllint-enable warning update-where */ + + INSERT #ActivitiesStaging + ( + "user_id", + "date", + "Yammer Posted", + "Yammer Read", + "Yammer Liked" ) SELECT - user_id, date, - [posted_count], [read_count], [liked_count] - FROM @yammer as tvp - WHERE NOT EXISTS (SELECT 1 FROM #ActivitiesStaging as t WHERE t.user_id = tvp.user_id AND t.date = tvp.date); -END + "user_id", + "date", + posted_count, + read_count, + liked_count + FROM @yammer AS tvp + WHERE + NOT EXISTS ( + SELECT 1 + FROM #ActivitiesStaging AS t + WHERE t."user_id" = tvp."user_id" AND t."date" = tvp."date" + ); +END; GO -CREATE PROCEDURE [profiling].[usp_UpsertTeamsDevices] ( - @StartDate DATE, @EndDate DATE -) AS +CREATE PROCEDURE profiling.usp_UpsertTeamsDevices +( + @StartDate DATE, + @EndDate DATE +) +AS BEGIN SET NOCOUNT ON; DECLARE @ut_staging AS ut_teams_user_device_usage_log; - - INSERT INTO @ut_staging ( - [user_id], [date], - [used_web], [used_mac], [used_windows], [used_linux], [used_chrome_os], - [used_mobile], [used_win_phone], [used_ios], [used_android] + INSERT INTO @ut_staging + ( + "user_id", + "date", + used_web, + used_mac, + used_windows, + used_linux, + used_chrome_os, + used_mobile, + used_win_phone, + used_ios, + used_android ) SELECT - [user_id], @StartDate, - -- multiplication here converts the bit column into an int so that aggregation works - MAX(1*[used_web]), MAX(1*[used_mac]), MAX(1*[used_windows]), + "user_id", + @StartDate, + -- multiplication here converts the BIT column into an INT so the aggregation works + MAX(1 * used_web), + MAX(1 * used_mac), + MAX(1 * used_windows), -- Use COALESCE here to fix #1 - MAX(1*COALESCE([used_linux], 0)), MAX(1*COALESCE([used_chrome_os], 0)), - MAX(1*[used_win_phone] + 1*[used_ios] + 1*[used_android]), -- used mobile - MAX(1*[used_win_phone]), MAX(1*[used_ios]), MAX(1*[used_android]) - FROM dbo.[teams_user_device_usage_log] - WHERE @StartDate <= [date] AND [date] <= @EndDate - GROUP BY [user_id]; - + MAX(1 * COALESCE(used_linux, 0)), + MAX(1 * COALESCE(used_chrome_os, 0)), + MAX(1 * used_win_phone + 1 * used_ios + 1 * used_android), + -- used mobile + MAX(1 * used_win_phone), + MAX(1 * used_ios), + MAX(1 * used_android) + FROM dbo.teams_user_device_usage_log + WHERE @StartDate <= "date" AND "date" <= @EndDate + GROUP BY "user_id"; + + /* tsqllint-disable warning update-where */ UPDATE t WITH (UPDLOCK, SERIALIZABLE) SET - [Teams Used Web] = staging.[used_web], - [Teams Used Mac] = staging.[used_mac], - [Teams Used Windows] = staging.[used_windows], - [Teams Used Linux] = staging.[used_linux], - [Teams Used Chrome OS] = staging.[used_chrome_os], - [Teams Used Mobile] = staging.[used_mobile], - [Teams Used WinPhone] = staging.[used_win_phone], - [Teams Used iOS] = staging.[used_ios], - [Teams Used Android] = staging.[used_android] + "Teams Used Web" = staging.used_web, + "Teams Used Mac" = staging.used_mac, + "Teams Used Windows" = staging.used_windows, + "Teams Used Linux" = staging.used_linux, + "Teams Used Chrome OS" = staging.used_chrome_os, + "Teams Used Mobile" = staging.used_mobile, + "Teams Used WinPhone" = staging.used_win_phone, + "Teams Used iOS" = staging.used_ios, + "Teams Used Android" = staging.used_android FROM #UsageStaging AS t - INNER JOIN @ut_staging AS staging ON t.user_id = staging.user_id AND t.date = staging.date; - - INSERT #UsageStaging ( - [user_id], [date], - [Teams Used Web], [Teams Used Mac], [Teams Used Windows], [Teams Used Linux], [Teams Used Chrome OS], - [Teams Used Mobile], [Teams Used WinPhone], [Teams Used iOS], [Teams Used Android] + INNER JOIN @ut_staging AS staging + ON t."user_id" = staging."user_id" AND t."date" = staging."date"; + /* tsqllint-enable warning update-where */ + + INSERT #UsageStaging + ( + "user_id", + "date", + "Teams Used Web", + "Teams Used Mac", + "Teams Used Windows", + "Teams Used Linux", + "Teams Used Chrome OS", + "Teams Used Mobile", + "Teams Used WinPhone", + "Teams Used iOS", + "Teams Used Android" ) SELECT - [user_id], [date], - [used_web], [used_mac], [used_windows], [used_linux], [used_chrome_os], - [used_mobile], [used_win_phone], [used_ios], [used_android] - FROM @ut_staging as staging - WHERE NOT EXISTS (SELECT 1 FROM #UsageStaging as t WHERE t.user_id = staging.user_id AND t.date = staging.date); -END + "user_id", + "date", + used_web, + used_mac, + used_windows, + used_linux, + used_chrome_os, + used_mobile, + used_win_phone, + used_ios, + used_android + FROM @ut_staging AS staging + WHERE NOT EXISTS ( + SELECT 1 + FROM #UsageStaging AS t + WHERE t."user_id" = staging."user_id" AND t."date" = staging."date" + ); +END; GO -CREATE PROCEDURE [profiling].[usp_UpsertM365Apps] ( - @StartDate DATE, @EndDate DATE -) AS +CREATE PROCEDURE profiling.usp_UpsertM365Apps +( + @StartDate DATE, + @EndDate DATE +) +AS BEGIN SET NOCOUNT ON; DECLARE @ut_staging AS ut_platform_user_activity_log; - - INSERT INTO @ut_staging ( - [user_id], [date], - [windows], [mac], [mobile], [web], - [outlook], [word], [excel], [powerpoint], [onenote], [teams], - [outlook_windows], [word_windows], [excel_windows], [powerpoint_windows], [onenote_windows], [teams_windows], - [outlook_mac], [word_mac], [excel_mac], [powerpoint_mac], [onenote_mac], [teams_mac], - [outlook_mobile], [word_mobile], [excel_mobile], [powerpoint_mobile], [onenote_mobile], [teams_mobile], - [outlook_web], [word_web], [excel_web], [powerpoint_web], [onenote_web], [teams_web] + INSERT INTO @ut_staging + ( + "user_id", + "date", + windows, + mac, + mobile, + web, + outlook, + word, + excel, + powerpoint, + onenote, + teams, + outlook_windows, + word_windows, + excel_windows, + powerpoint_windows, + onenote_windows, + teams_windows, + outlook_mac, + word_mac, + excel_mac, + powerpoint_mac, + onenote_mac, + teams_mac, + outlook_mobile, + word_mobile, + excel_mobile, + powerpoint_mobile, + onenote_mobile, + teams_mobile, + outlook_web, + word_web, + excel_web, + powerpoint_web, + onenote_web, + teams_web ) SELECT - [user_id], @StartDate, - MAX(1*[windows]), MAX(1*[mac]), MAX(1*[mobile]), MAX(1*[web]), - MAX(1*[outlook]), MAX(1*[word]), MAX(1*[excel]), MAX(1*[powerpoint]), MAX(1*[onenote]), MAX(1*[teams]), - MAX(1*[outlook_windows]), MAX(1*[word_windows]), MAX(1*[excel_windows]), MAX(1*[powerpoint_windows]), MAX(1*[onenote_windows]), MAX(1*[teams_windows]), - MAX(1*[outlook_mac]), MAX(1*[word_mac]), MAX(1*[excel_mac]), MAX(1*[powerpoint_mac]), MAX(1*[onenote_mac]), MAX(1*[teams_mac]), - MAX(1*[outlook_mobile]), MAX(1*[word_mobile]), MAX(1*[excel_mobile]), MAX(1*[powerpoint_mobile]), MAX(1*[onenote_mobile]), MAX(1*[teams_mobile]), - MAX(1*[outlook_web]), MAX(1*[word_web]), MAX(1*[excel_web]), MAX(1*[powerpoint_web]), MAX(1*[onenote_web]), MAX(1*[teams_web]) - FROM dbo.[platform_user_activity_log] - WHERE @StartDate <= [date] AND [date] <= @EndDate - GROUP BY [user_id]; - + "user_id", + @StartDate, + MAX(1 * windows), + MAX(1 * mac), + MAX(1 * mobile), + MAX(1 * web), + MAX(1 * outlook), + MAX(1 * word), + MAX(1 * excel), + MAX(1 * powerpoint), + MAX(1 * onenote), + MAX(1 * teams), + MAX(1 * outlook_windows), + MAX(1 * word_windows), + MAX(1 * excel_windows), + MAX(1 * powerpoint_windows), + MAX(1 * onenote_windows), + MAX(1 * teams_windows), + MAX(1 * outlook_mac), + MAX(1 * word_mac), + MAX(1 * excel_mac), + MAX(1 * powerpoint_mac), + MAX(1 * onenote_mac), + MAX(1 * teams_mac), + MAX(1 * outlook_mobile), + MAX(1 * word_mobile), + MAX(1 * excel_mobile), + MAX(1 * powerpoint_mobile), + MAX(1 * onenote_mobile), + MAX(1 * teams_mobile), + MAX(1 * outlook_web), + MAX(1 * word_web), + MAX(1 * excel_web), + MAX(1 * powerpoint_web), + MAX(1 * onenote_web), + MAX(1 * teams_web) + FROM dbo.platform_user_activity_log + WHERE @StartDate <= "date" AND "date" <= @EndDate + GROUP BY "user_id"; + + /* tsqllint-disable warning update-where */ UPDATE t WITH (UPDLOCK, SERIALIZABLE) SET - [Office Windows] = staging.[windows], - [Office Mac] = staging.[mac], - [Office Mobile] = staging.[mobile], - [Office Web] = staging.[web], - [Office Outlook] = staging.[outlook], - [Office Word] = staging.[word], - [Office Excel] = staging.[excel], - [Office Powerpoint] = staging.[powerpoint], - [Office Onenote] = staging.[onenote], - [Office Teams] = staging.[teams], - [Office Outlook Windows] = staging.[outlook_windows], - [Office Word Windows] = staging.[word_windows], - [Office Excel Windows] = staging.[excel_windows], - [Office Powerpoint Windows] = staging.[powerpoint_windows], - [Office Onenote Windows] = staging.[onenote_windows], - [Office Teams Windows] = staging.[teams_windows], - [Office Outlook Mac] = staging.[outlook_mac], - [Office Word Mac] = staging.[word_mac], - [Office Excel Mac] = staging.[excel_mac], - [Office Powerpoint Mac] = staging.[powerpoint_mac], - [Office Onenote Mac] = staging.[onenote_mac], - [Office Teams Mac] = staging.[teams_mac], - [Office Outlook Mobile] = staging.[outlook_mobile], - [Office Word Mobile] = staging.[word_mobile], - [Office Excel Mobile] = staging.[excel_mobile], - [Office Powerpoint Mobile] = staging.[powerpoint_mobile], - [Office Onenote Mobile] = staging.[onenote_mobile], - [Office Teams Mobile] = staging.[teams_mobile], - [Office Outlook Web] = staging.[outlook_web], - [Office Word Web] = staging.[word_web], - [Office Excel Web] = staging.[excel_web], - [Office Powerpoint Web] = staging.[powerpoint_web], - [Office Onenote Web] = staging.[onenote_web], - [Office Teams Web] = staging.[teams_web] + "Office Windows" = staging.windows, + "Office Mac" = staging.mac, + "Office Mobile" = staging.mobile, + "Office Web" = staging.web, + "Office Outlook" = staging.outlook, + "Office Word" = staging.word, + "Office Excel" = staging.excel, + "Office Powerpoint" = staging.powerpoint, + "Office Onenote" = staging.onenote, + "Office Teams" = staging.teams, + "Office Outlook Windows" = staging.outlook_windows, + "Office Word Windows" = staging.word_windows, + "Office Excel Windows" = staging.excel_windows, + "Office Powerpoint Windows" = staging.powerpoint_windows, + "Office Onenote Windows" = staging.onenote_windows, + "Office Teams Windows" = staging.teams_windows, + "Office Outlook Mac" = staging.outlook_mac, + "Office Word Mac" = staging.word_mac, + "Office Excel Mac" = staging.excel_mac, + "Office Powerpoint Mac" = staging.powerpoint_mac, + "Office Onenote Mac" = staging.onenote_mac, + "Office Teams Mac" = staging.teams_mac, + "Office Outlook Mobile" = staging.outlook_mobile, + "Office Word Mobile" = staging.word_mobile, + "Office Excel Mobile" = staging.excel_mobile, + "Office Powerpoint Mobile" = staging.powerpoint_mobile, + "Office Onenote Mobile" = staging.onenote_mobile, + "Office Teams Mobile" = staging.teams_mobile, + "Office Outlook Web" = staging.outlook_web, + "Office Word Web" = staging.word_web, + "Office Excel Web" = staging.excel_web, + "Office Powerpoint Web" = staging.powerpoint_web, + "Office Onenote Web" = staging.onenote_web, + "Office Teams Web" = staging.teams_web FROM #UsageStaging AS t - INNER JOIN @ut_staging AS staging ON t.user_id = staging.user_id AND t.date = staging.date; - - INSERT #UsageStaging ( - [user_id], [date], - [Office Windows], [Office Mac], [Office Mobile], [Office Web], - [Office Outlook], [Office Word], [Office Excel], [Office Powerpoint], [Office Onenote], [Office Teams], - [Office Outlook Windows], [Office Word Windows], [Office Excel Windows], [Office Powerpoint Windows], [Office Onenote Windows], [Office Teams Windows], - [Office Outlook Mac], [Office Word Mac], [Office Excel Mac], [Office Powerpoint Mac], [Office Onenote Mac], [Office Teams Mac], - [Office Outlook Mobile], [Office Word Mobile], [Office Excel Mobile], [Office Powerpoint Mobile], [Office Onenote Mobile], [Office Teams Mobile], - [Office Outlook Web], [Office Word Web], [Office Excel Web], [Office Powerpoint Web], [Office Onenote Web], [Office Teams Web] + INNER JOIN @ut_staging AS staging ON t."user_id" = staging."user_id" + AND t."date" = staging."date"; + /* tsqllint-enable warning update-where */ + + INSERT #UsageStaging + ( + "user_id", + "date", + "Office Windows", + "Office Mac", + "Office Mobile", + "Office Web", + "Office Outlook", + "Office Word", + "Office Excel", + "Office Powerpoint", + "Office Onenote", + "Office Teams", + "Office Outlook Windows", + "Office Word Windows", + "Office Excel Windows", + "Office Powerpoint Windows", + "Office Onenote Windows", + "Office Teams Windows", + "Office Outlook Mac", + "Office Word Mac", + "Office Excel Mac", + "Office Powerpoint Mac", + "Office Onenote Mac", + "Office Teams Mac", + "Office Outlook Mobile", + "Office Word Mobile", + "Office Excel Mobile", + "Office Powerpoint Mobile", + "Office Onenote Mobile", + "Office Teams Mobile", + "Office Outlook Web", + "Office Word Web", + "Office Excel Web", + "Office Powerpoint Web", + "Office Onenote Web", + "Office Teams Web" ) SELECT - [user_id], [date], - [windows], [mac], [mobile], [web], - [outlook], [word], [excel], [powerpoint], [onenote], [teams], - [outlook_windows], [word_windows], [excel_windows], [powerpoint_windows], [onenote_windows], [teams_windows], - [outlook_mac], [word_mac], [excel_mac], [powerpoint_mac], [onenote_mac], [teams_mac], - [outlook_mobile], [word_mobile], [excel_mobile], [powerpoint_mobile], [onenote_mobile], [teams_mobile], - [outlook_web], [word_web], [excel_web], [powerpoint_web], [onenote_web], [teams_web] - FROM @ut_staging as staging - WHERE NOT EXISTS (SELECT 1 FROM #UsageStaging as t WHERE t.user_id = staging.user_id AND t.date = staging.date); -END + "user_id", + "date", + windows, + mac, + mobile, + web, + outlook, + word, + excel, + powerpoint, + onenote, + teams, + outlook_windows, + word_windows, + excel_windows, + powerpoint_windows, + onenote_windows, + teams_windows, + outlook_mac, + word_mac, + excel_mac, + powerpoint_mac, + onenote_mac, + teams_mac, + outlook_mobile, + word_mobile, + excel_mobile, + powerpoint_mobile, + onenote_mobile, + teams_mobile, + outlook_web, + word_web, + excel_web, + powerpoint_web, + onenote_web, + teams_web + FROM @ut_staging AS staging + WHERE NOT EXISTS ( + SELECT 1 + FROM #UsageStaging AS t + WHERE t."user_id" = staging."user_id" AND t."date" = staging."date" + ); +END; GO -CREATE PROCEDURE [profiling].[usp_UpsertYammerDevices] ( - @StartDate DATE, @EndDate DATE -) AS +CREATE PROCEDURE profiling.usp_UpsertYammerDevices +( + @StartDate DATE, + @EndDate DATE +) +AS BEGIN SET NOCOUNT ON; DECLARE @ut_staging AS ut_yammer_device_activity_log; - - INSERT INTO @ut_staging ( - [user_id], [date], - [used_count], [used_web], [used_mobile], [used_others], - [used_win_phone], [used_android], [used_ipad], [used_iphone] + INSERT INTO @ut_staging + ( + "user_id", + "date", + used_count, + used_web, + used_mobile, + used_others, + used_win_phone, + used_android, + used_ipad, + used_iphone ) SELECT - [user_id], @StartDate, - MAX(1 * [used_web] + 1 * [used_others] + 1 * [used_win_phone] + 1 * [used_android] - + 1 * [used_ipad] + 1 * [used_iphone] + 1 * [used_others]), - MAX(1 * [used_web]), - MAX(1 * [used_win_phone] + 1 * [used_android] + 1 * [used_ipad] + 1 * [used_iphone]), - MAX(1 * [used_others]), - MAX(1 * [used_win_phone]), - MAX(1 * [used_android]), - MAX(1 * [used_ipad]), - MAX(1 * [used_iphone]) - FROM [dbo].[yammer_device_activity_log] - WHERE @StartDate <= [date] AND [date] <= @EndDate - GROUP BY [user_id]; - + "user_id", + @StartDate, + MAX(1 * used_web + 1 * used_others + 1 * used_win_phone + 1 * used_android + + 1 * used_ipad + 1 * used_iphone + 1 * used_others), + MAX(1 * used_web), + MAX(1 * used_win_phone + 1 * used_android + 1 * used_ipad + 1 * used_iphone), + MAX(1 * used_others), + MAX(1 * used_win_phone), + MAX(1 * used_android), + MAX(1 * used_ipad), + MAX(1 * used_iphone) + FROM dbo.yammer_device_activity_log + WHERE @StartDate <= "date" AND "date" <= @EndDate + GROUP BY "user_id"; + + /* tsqllint-disable warning update-where */ UPDATE t WITH (UPDLOCK, SERIALIZABLE) SET - [Yammer Platform Count] = staging.[used_count], - [Yammer Used Web] = staging.[used_web], - [Yammer Used Mobile] = staging.[used_mobile], - [Yammer Used Others] = staging.[used_others], - [Yammer Used WinPhone] = staging.[used_win_phone], - [Yammer Used Android] = staging.[used_android], - [Yammer Used iPad] = staging.[used_ipad], - [Yammer Used iPhone] = staging.[used_iphone] + "Yammer Platform Count" = staging.used_count, + "Yammer Used Web" = staging.used_web, + "Yammer Used Mobile" = staging.used_mobile, + "Yammer Used Others" = staging.used_others, + "Yammer Used WinPhone" = staging.used_win_phone, + "Yammer Used Android" = staging.used_android, + "Yammer Used iPad" = staging.used_ipad, + "Yammer Used iPhone" = staging.used_iphone FROM #UsageStaging AS t - INNER JOIN @ut_staging AS staging ON t.user_id = staging.user_id AND t.date = staging.date; + INNER JOIN @ut_staging AS staging + ON t."user_id" = staging."user_id" AND t."date" = staging."date"; + /* tsqllint-enable warning update-where */ + + INSERT #UsageStaging + ( + "user_id", + "date", + "Yammer Platform Count", + "Yammer Used Web", + "Yammer Used Mobile", + "Yammer Used Others", + "Yammer Used WinPhone", + "Yammer Used Android", + "Yammer Used iPad", + "Yammer Used iPhone" + ) + SELECT + "user_id", + "date", + used_count, + used_web, + used_mobile, + used_others, + used_win_phone, + used_android, + used_ipad, + used_iphone + FROM @ut_staging AS staging + WHERE NOT EXISTS ( + SELECT 1 + FROM #UsageStaging AS t + WHERE t."user_id" = staging."user_id" AND t."date" = staging."date" + ); +END; +GO - INSERT #UsageStaging ( - user_id, date, - [Yammer Platform Count], [Yammer Used Web], [Yammer Used Mobile], [Yammer Used Others], - [Yammer Used WinPhone], [Yammer Used Android], [Yammer Used iPad], [Yammer Used iPhone] +CREATE PROCEDURE profiling.usp_UpsertCopilot +( + @StartDate DATE, + @EndDate DATE +) +AS +BEGIN + SET NOCOUNT ON; + DECLARE @copilot AS ut_copilot_activities; + WITH + hosts_pivoted AS ( + SELECT + "user_id", + "date", + bizchat, + appchat, + Assist365, + Bing, + BashTool, + DevUI, + Excel, + Loop, + M365AdminCenter, + M365App, + Office, + OneNote, + Outlook, + Planner, + PowerPoint, + SharePoint, + Stream, + Teams, + VivaCopilot, + VivaEngage, + VivaGoals, + Whiteboard, + Word + FROM ( + SELECT + app_host, + @StartDate AS "date", + "user_id", + event_id + FROM dbo.event_copilot_chats AS c + JOIN dbo.audit_events AS au ON c.event_id = au.id + WHERE @StartDate <= au.time_stamp AND au.time_stamp <= @EndDate + ) t + PIVOT ( + COUNT(event_id) + FOR app_host IN ( + -- AS more hosts appear, they need to be added here AS they are in the JSON + -- bizchat is M365 Chat experience on Bing and Teams + -- appchat refers to the copilot experiences on apps like Excel, PowerPoint, Teams, and more + bizchat, + appchat, + Assist365, + Bing, + BashTool, + DevUI, + Excel, + Loop, + M365AdminCenter, + M365App, + Office, + OneNote, + Outlook, + Planner, + PowerPoint, + SharePoint, + Stream, + Teams, + VivaCopilot, + VivaEngage, + VivaGoals, + Whiteboard, + Word + ) + ) AS pivoted + ), + host_activities AS ( + SELECT + "user_id", + "date", + Assist365 AS copilot_assist365, + Bing AS copilot_bing, + BashTool AS copilot_bashtool, + DevUI AS copilot_devui, + Excel AS copilot_excel, + Loop AS copilot_loop, + M365AdminCenter AS copilot_m365admincenter, + (bizchat + M365App) AS copilot_m365app, + (appchat + Office) AS copilot_office, + OneNote AS copilot_onenote, + Outlook AS copilot_outlook, + Planner AS copilot_planner, + PowerPoint AS copilot_powerpoint, + SharePoint AS copilot_sharepoint, + Stream AS copilot_stream, + Teams AS copilot_teams, + VivaCopilot AS copilot_vivacopilot, + VivaEngage AS copilot_vivaengage, + VivaGoals AS copilot_vivagoals, + Whiteboard AS copilot_whiteboard, + Word AS copilot_word + FROM hosts_pivoted + ), + events AS ( + SELECT + "user_id", + @StartDate AS "date", + c.event_id AS chat_id, + f.copilot_chat_id AS has_file, + m.copilot_chat_id AS has_meeting + FROM dbo.event_copilot_chats AS c + JOIN dbo.audit_events AS au ON c.event_id = au.id + LEFT JOIN dbo.event_copilot_files AS f ON c.event_id = f.copilot_chat_id + LEFT JOIN dbo.event_copilot_meetings AS m ON c.event_id = m.copilot_chat_id + WHERE @StartDate <= au.time_stamp AND au.time_stamp <= @EndDate + ), + event_counts AS ( + SELECT + "user_id", + "date", + COUNT(chat_id) AS chat_count, + COUNT(has_file) AS file_count, + COUNT(has_meeting) AS meeting_count + FROM events + GROUP BY "user_id", "date" + ) + INSERT INTO @copilot + ( + "user_id", + "date", + copilot_chats, + copilot_meetings, + copilot_files, + copilot_assist365, + copilot_bing, + copilot_bashtool, + copilot_devui, + copilot_excel, + copilot_loop, + copilot_m365admincenter, + copilot_m365app, + copilot_office, + copilot_onenote, + copilot_outlook, + copilot_planner, + copilot_powerpoint, + copilot_sharepoint, + copilot_stream, + copilot_teams, + copilot_vivacopilot, + copilot_vivaengage, + copilot_vivagoals, + copilot_whiteboard, + copilot_word ) SELECT - user_id, date, - [used_count], [used_web], [used_mobile], [used_others], - [used_win_phone], [used_android], [used_ipad], [used_iphone] - FROM @ut_staging as staging - WHERE NOT EXISTS (SELECT 1 FROM #UsageStaging as t WHERE t.user_id = staging.user_id AND t.date = staging.date); -END + a."user_id", + @StartDate, + SUM(c.chat_count), + SUM(c.meeting_count), + SUM(c.file_count), + SUM(copilot_assist365), + SUM(copilot_bing), + SUM(copilot_bashtool), + SUM(copilot_devui), + SUM(copilot_excel), + SUM(copilot_loop), + SUM(copilot_m365admincenter), + SUM(copilot_m365app), + SUM(copilot_office), + SUM(copilot_onenote), + SUM(copilot_outlook), + SUM(copilot_planner), + SUM(copilot_powerpoint), + SUM(copilot_sharepoint), + SUM(copilot_stream), + SUM(copilot_teams), + SUM(copilot_vivacopilot), + SUM(copilot_vivaengage), + SUM(copilot_vivagoals), + SUM(copilot_whiteboard), + SUM(copilot_word) + FROM host_activities AS a + JOIN event_counts AS c ON a."user_id" = c."user_id" + GROUP BY a."user_id"; + + /* tsqllint-disable warning update-where */ + UPDATE t WITH (UPDLOCK, SERIALIZABLE) + SET + "Copilot Chats" = tvp.copilot_chats, + "Copilot Meetings" = tvp.copilot_meetings, + "Copilot Files" = tvp.copilot_files, + "Copilot App Assist365" = tvp.copilot_assist365, + "Copilot App Bing" = tvp.copilot_bing, + "Copilot App BashTool" = tvp.copilot_bashtool, + "Copilot App DevUI" = tvp.copilot_devui, + "Copilot App Excel" = tvp.copilot_excel, + "Copilot App Loop" = tvp.copilot_loop, + "Copilot App M365AdminCenter" = tvp.copilot_m365admincenter, + "Copilot App M365App" = tvp.copilot_m365app, + "Copilot App Office" = tvp.copilot_office, + "Copilot App OneNote" = tvp.copilot_onenote, + "Copilot App Outlook" = tvp.copilot_outlook, + "Copilot App Planner" = tvp.copilot_planner, + "Copilot App PowerPoint" = tvp.copilot_powerpoint, + "Copilot App SharePoint" = tvp.copilot_sharepoint, + "Copilot App Stream" = tvp.copilot_stream, + "Copilot App Teams" = tvp.copilot_teams, + "Copilot App VivaCopilot" = tvp.copilot_vivacopilot, + "Copilot App VivaEngage" = tvp.copilot_vivaengage, + "Copilot App VivaGoals" = tvp.copilot_vivagoals, + "Copilot App Whiteboard" = tvp.copilot_whiteboard, + "Copilot App Word" = tvp.copilot_word + FROM #ActivitiesStaging AS t + INNER JOIN @copilot AS tvp + ON t."user_id" = tvp."user_id" AND t."date" = tvp."date"; + /* tsqllint-enable warning update-where */ + + INSERT #ActivitiesStaging + ( + "user_id", + "date", + "Copilot Chats", + "Copilot Meetings", + "Copilot Files", + "Copilot App Assist365", + "Copilot App Bing", + "Copilot App BashTool", + "Copilot App DevUI", + "Copilot App Excel", + "Copilot App Loop", + "Copilot App M365AdminCenter", + "Copilot App M365App", + "Copilot App Office", + "Copilot App OneNote", + "Copilot App Outlook", + "Copilot App Planner", + "Copilot App PowerPoint", + "Copilot App SharePoint", + "Copilot App Stream", + "Copilot App Teams", + "Copilot App VivaCopilot", + "Copilot App VivaEngage", + "Copilot App VivaGoals", + "Copilot App Whiteboard", + "Copilot App Word" + ) + SELECT + "user_id", + "date", + copilot_chats, + copilot_meetings, + copilot_files, + copilot_assist365, + copilot_bing, + copilot_bashtool, + copilot_devui, + copilot_excel, + copilot_loop, + copilot_m365admincenter, + copilot_m365app, + copilot_office, + copilot_onenote, + copilot_outlook, + copilot_planner, + copilot_powerpoint, + copilot_sharepoint, + copilot_stream, + copilot_teams, + copilot_vivacopilot, + copilot_vivaengage, + copilot_vivagoals, + copilot_whiteboard, + copilot_word + FROM @copilot AS tvp + WHERE NOT EXISTS ( + SELECT 1 + FROM #ActivitiesStaging AS t + WHERE t."user_id" = tvp."user_id" AND t."date" = tvp."date" + ); +END; GO -- =========================================== -- Aggregates a week of activity. Data in rows -- =========================================== -IF OBJECT_ID(N'profiling.usp_CompileWeekActivityRows') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileWeekActivityRows]; + +IF OBJECT_ID(N'profiling.usp_CompileWeekActivityRows') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileWeekActivityRows; +END GO -CREATE PROCEDURE [profiling].[usp_CompileWeekActivityRows] ( - @Monday DATE -- Start day of the week to aggregate -) AS +CREATE PROCEDURE profiling.usp_CompileWeekActivityRows +( + -- Start day of the week to aggregate + @Monday DATE +) +AS BEGIN SET NOCOUNT ON; BEGIN TRY - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'[usp_CompileWeekActivityRows] Starting: ' + CAST(@Monday AS NVARCHAR); - INSERT INTO [profiling].[ActivitiesWeekly] + EXEC profiling.usp_Trace '[usp_CompileWeekActivityRows] Starting: %s', @Monday; + + INSERT INTO profiling.ActivitiesWeekly SELECT - user_id, - @Monday as MetricDate, + "user_id", + @Monday AS MetricDate, Metric, - Sum(Value) as Sum + SUM(VALUE) AS Sum FROM #ActivitiesStaging AS Pivoted UNPIVOT ( -- Convert columns into rows - Value FOR Metric IN ( - [OneDrive Viewed/Edited], [OneDrive Synced], - [OneDrive Shared Internally], [OneDrive Shared Externally], - [Emails Sent], [Emails Received], [Emails Read], - [Outlook Meetings Created], [Outlook Meetings Interacted], - [SPO Viewed/Edited], [SPO Synced], - [SPO Shared Internally], [SPO Shared Externally], - [Teams Private Chats], [Teams Team Chats], [Teams Calls], - [Teams Meetings], [Teams Meetings Attended], [Teams Meetings Organized], - [Teams Adhoc Meetings Attended], [Teams Adhoc Meetings Organized], - [Teams Scheduled Onetime Meetings Attended], [Teams Scheduled Onetime Meetings Organized], - [Teams Scheduled Recurring Meetings Attended], [Teams Scheduled Recurring Meetings Organized], - [Teams Audio Duration Seconds], [Teams Video Duration Seconds], [Teams Screenshare Duration Seconds], - [Teams Post Messages], [Teams Reply Messages], [Teams Urgent Messages], - [Yammer Posted], [Yammer Read], [Yammer Liked] + VALUE FOR Metric IN ( + "OneDrive Viewed/Edited", + "OneDrive Synced", + "OneDrive Shared Internally", + "OneDrive Shared Externally", + "Emails Sent", + "Emails Received", + "Emails Read", + "Outlook Meetings Created", + "Outlook Meetings Interacted", + "SPO Viewed/Edited", + "SPO Synced", + "SPO Shared Internally", + "SPO Shared Externally", + "Teams Private Chats", + "Teams Team Chats", + "Teams Calls", + "Teams Meetings", + "Teams Meetings Attended", + "Teams Meetings Organized", + "Teams Adhoc Meetings Attended", + "Teams Adhoc Meetings Organized", + "Teams Scheduled Onetime Meetings Attended", + "Teams Scheduled Onetime Meetings Organized", + "Teams Scheduled Recurring Meetings Attended", + "Teams Scheduled Recurring Meetings Organized", + "Teams Audio Duration Seconds", + "Teams Video Duration Seconds", + "Teams Screenshare Duration Seconds", + "Teams Post Messages", + "Teams Reply Messages", + "Teams Urgent Messages", + "Yammer Posted", + "Yammer Read", + "Yammer Liked", + "Copilot Chats", + "Copilot Meetings", + "Copilot Files", + "Copilot App Assist365", + "Copilot App Bing", + "Copilot App BashTool", + "Copilot App DevUI", + "Copilot App Excel", + "Copilot App Loop", + "Copilot App M365AdminCenter", + "Copilot App M365App", + "Copilot App Office", + "Copilot App OneNote", + "Copilot App Outlook", + "Copilot App Planner", + "Copilot App PowerPoint", + "Copilot App SharePoint", + "Copilot App Stream", + "Copilot App Teams", + "Copilot App VivaCopilot", + "Copilot App VivaEngage", + "Copilot App VivaGoals", + "Copilot App Whiteboard", + "Copilot App Word" ) ) AS Unpivoted - GROUP BY user_id, Metric; + GROUP BY "user_id", Metric; END TRY BEGIN CATCH - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'[usp_CompileWeekActivityRows] Catch: ' + ERROR_MESSAGE(); - END CATCH -END + DECLARE @ErrorMessage NVARCHAR(4000); + SELECT @ErrorMessage = ERROR_MESSAGE(); + EXEC profiling.usp_Trace '[usp_CompileWeekActivityRows] Catch: %s', @ErrorMessage; + END CATCH; +END; GO -- ============================================== -- Aggregates a week of activity. Data in columns -- ============================================== -IF OBJECT_ID(N'profiling.usp_CompileWeekActivityColumns') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileWeekActivityColumns]; + +IF OBJECT_ID(N'profiling.usp_CompileWeekActivityColumns') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileWeekActivityColumns; +END GO -CREATE PROCEDURE [profiling].[usp_CompileWeekActivityColumns] ( - @Monday DATE -- Start day of the week to aggregate -) AS +CREATE PROCEDURE profiling.usp_CompileWeekActivityColumns +( + -- Start day of the week to aggregate + @Monday DATE +) +AS BEGIN SET NOCOUNT ON; BEGIN TRY - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'[usp_CompileWeekActivityColumns] Starting: ' + CAST(@Monday AS NVARCHAR); - INSERT INTO [profiling].[ActivitiesWeeklyColumns] ( - user_id, date, - [OneDrive Viewed/Edited], [OneDrive Synced], - [OneDrive Shared Internally], [OneDrive Shared Externally], - - [Emails Sent], [Emails Received], [Emails Read], - [Outlook Meetings Created], [Outlook Meetings Interacted], - - [SPO Viewed/Edited], [SPO Synced], - [SPO Shared Internally], [SPO Shared Externally], - - [Teams Private Chats], [Teams Team Chats], [Teams Calls], - [Teams Meetings], [Teams Meetings Attended], [Teams Meetings Organized], - [Teams Adhoc Meetings Attended], [Teams Adhoc Meetings Organized], - [Teams Scheduled Onetime Meetings Attended], [Teams Scheduled Onetime Meetings Organized], - [Teams Scheduled Recurring Meetings Attended], [Teams Scheduled Recurring Meetings Organized], - [Teams Audio Duration Seconds], [Teams Video Duration Seconds], [Teams Screenshare Duration Seconds], - [Teams Post Messages], [Teams Reply Messages], [Teams Urgent Messages], - - [Yammer Posted], [Yammer Read], [Yammer Liked] + EXEC profiling.usp_Trace '[usp_CompileWeekActivityColumns] Starting: %s', @Monday; + + INSERT INTO profiling.ActivitiesWeeklyColumns + ( + "user_id", + "date", + "OneDrive Viewed/Edited", + "OneDrive Synced", + "OneDrive Shared Internally", + "OneDrive Shared Externally", + "Emails Sent", + "Emails Received", + "Emails Read", + "Outlook Meetings Created", + "Outlook Meetings Interacted", + "SPO Viewed/Edited", + "SPO Synced", + "SPO Shared Internally", + "SPO Shared Externally", + "Teams Private Chats", + "Teams Team Chats", + "Teams Calls", + "Teams Meetings", + "Teams Meetings Attended", + "Teams Meetings Organized", + "Teams Adhoc Meetings Attended", + "Teams Adhoc Meetings Organized", + "Teams Scheduled Onetime Meetings Attended", + "Teams Scheduled Onetime Meetings Organized", + "Teams Scheduled Recurring Meetings Attended", + "Teams Scheduled Recurring Meetings Organized", + "Teams Audio Duration Seconds", + "Teams Video Duration Seconds", + "Teams Screenshare Duration Seconds", + "Teams Post Messages", + "Teams Reply Messages", + "Teams Urgent Messages", + "Yammer Posted", + "Yammer Read", + "Yammer Liked", + "Copilot Chats", + "Copilot Meetings", + "Copilot Files", + "Copilot App Assist365", + "Copilot App Bing", + "Copilot App BashTool", + "Copilot App DevUI", + "Copilot App Excel", + "Copilot App Loop", + "Copilot App M365AdminCenter", + "Copilot App M365App", + "Copilot App Office", + "Copilot App OneNote", + "Copilot App Outlook", + "Copilot App Planner", + "Copilot App PowerPoint", + "Copilot App SharePoint", + "Copilot App Stream", + "Copilot App Teams", + "Copilot App VivaCopilot", + "Copilot App VivaEngage", + "Copilot App VivaGoals", + "Copilot App Whiteboard", + "Copilot App Word" ) SELECT - user_id, - @Monday as date, - [OneDrive Viewed/Edited], [OneDrive Synced], - [OneDrive Shared Internally], [OneDrive Shared Externally], - - [Emails Sent], [Emails Received], [Emails Read], - [Outlook Meetings Created], [Outlook Meetings Interacted], - - [SPO Viewed/Edited], [SPO Synced], - [SPO Shared Internally], [SPO Shared Externally], - - [Teams Private Chats], [Teams Team Chats], [Teams Calls], - [Teams Meetings], [Teams Meetings Attended], [Teams Meetings Organized], - [Teams Adhoc Meetings Attended], [Teams Adhoc Meetings Organized], - [Teams Scheduled Onetime Meetings Attended], [Teams Scheduled Onetime Meetings Organized], - [Teams Scheduled Recurring Meetings Attended], [Teams Scheduled Recurring Meetings Organized], - [Teams Audio Duration Seconds], [Teams Video Duration Seconds], [Teams Screenshare Duration Seconds], - [Teams Post Messages], [Teams Reply Messages], [Teams Urgent Messages], - - [Yammer Posted], [Yammer Read], [Yammer Liked] + "user_id", + @Monday AS "date", + "OneDrive Viewed/Edited", + "OneDrive Synced", + "OneDrive Shared Internally", + "OneDrive Shared Externally", + "Emails Sent", + "Emails Received", + "Emails Read", + "Outlook Meetings Created", + "Outlook Meetings Interacted", + "SPO Viewed/Edited", + "SPO Synced", + "SPO Shared Internally", + "SPO Shared Externally", + "Teams Private Chats", + "Teams Team Chats", + "Teams Calls", + "Teams Meetings", + "Teams Meetings Attended", + "Teams Meetings Organized", + "Teams Adhoc Meetings Attended", + "Teams Adhoc Meetings Organized", + "Teams Scheduled Onetime Meetings Attended", + "Teams Scheduled Onetime Meetings Organized", + "Teams Scheduled Recurring Meetings Attended", + "Teams Scheduled Recurring Meetings Organized", + "Teams Audio Duration Seconds", + "Teams Video Duration Seconds", + "Teams Screenshare Duration Seconds", + "Teams Post Messages", + "Teams Reply Messages", + "Teams Urgent Messages", + "Yammer Posted", + "Yammer Read", + "Yammer Liked", + "Copilot Chats", + "Copilot Meetings", + "Copilot Files", + "Copilot App Assist365", + "Copilot App Bing", + "Copilot App BashTool", + "Copilot App DevUI", + "Copilot App Excel", + "Copilot App Loop", + "Copilot App M365AdminCenter", + "Copilot App M365App", + "Copilot App Office", + "Copilot App OneNote", + "Copilot App Outlook", + "Copilot App Planner", + "Copilot App PowerPoint", + "Copilot App SharePoint", + "Copilot App Stream", + "Copilot App Teams", + "Copilot App VivaCopilot", + "Copilot App VivaEngage", + "Copilot App VivaGoals", + "Copilot App Whiteboard", + "Copilot App Word" FROM #ActivitiesStaging; END TRY BEGIN CATCH - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'[usp_CompileWeekActivityColumns] Catch: ' + ERROR_MESSAGE(); - END CATCH -END + DECLARE @ErrorMessage NVARCHAR(4000); + SELECT @ErrorMessage = ERROR_MESSAGE(); + EXEC profiling.usp_Trace '[usp_CompileWeekActivityColumns] Catch: %s', @ErrorMessage; + END CATCH; +END; GO -- =========================================== -- Aggregates a week of usage. Data in columns -- =========================================== -IF OBJECT_ID(N'profiling.usp_CompileUsageWeek') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileUsageWeek]; + +IF OBJECT_ID(N'profiling.usp_CompileUsageWeek') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileUsageWeek; +END GO -CREATE PROCEDURE [profiling].[usp_CompileUsageWeek] ( - @Monday DATE -- Start day of the week to aggregate -) AS +CREATE PROCEDURE profiling.usp_CompileUsageWeek +( + -- Start day of the week to aggregate + @Monday DATE +) +AS BEGIN SET NOCOUNT ON; BEGIN TRY - -- SET NOCOUNT ON added to prevent extra result sets from interfering with SELECT statements. - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'[usp_CompileUsageWeek] Starting: ' + CAST(@Monday AS NVARCHAR); + EXEC profiling.usp_Trace '[usp_CompileUsageWeek] Starting: %s', @Monday; DECLARE @Sunday DATE = DATEADD(DAY, 6, @Monday), @ColumnsDone INT = 0; - + -- Check if the data has already been aggregated - SELECT @ColumnsDone=COUNT([date]) - FROM [profiling].[UsageWeekly] - WHERE [date] = @Monday; - + SELECT @ColumnsDone = COUNT(DATE) + FROM profiling.UsageWeekly + WHERE "date" = @Monday; + -- If the data has not been aggregated, do it IF @ColumnsDone = 0 BEGIN - CREATE TABLE #UsageStaging ( - [user_id] [int] NOT NULL, - [date] [datetime] NOT NULL, - -- Teams devices - [Teams Used Web] [int] NOT NULL DEFAULT 0, - [Teams Used Mac] [int] NOT NULL DEFAULT 0, - [Teams Used Windows] [int] NOT NULL DEFAULT 0, - [Teams Used Linux] [int] NOT NULL DEFAULT 0, - [Teams Used Chrome OS] [int] NOT NULL DEFAULT 0, - [Teams Used Mobile] [int] NOT NULL DEFAULT 0, - [Teams Used WinPhone] [int] NOT NULL DEFAULT 0, - [Teams Used iOS] [int] NOT NULL DEFAULT 0, - [Teams Used Android] [int] NOT NULL DEFAULT 0, - -- M365 app - [Office Windows] [int] NOT NULL DEFAULT 0, - [Office Mac] [int] NOT NULL DEFAULT 0, - [Office Mobile] [int] NOT NULL DEFAULT 0, - [Office Web] [int] NOT NULL DEFAULT 0, - [Office Outlook] [int] NOT NULL DEFAULT 0, - [Office Word] [int] NOT NULL DEFAULT 0, - [Office Excel] [int] NOT NULL DEFAULT 0, - [Office Powerpoint] [int] NOT NULL DEFAULT 0, - [Office Onenote] [int] NOT NULL DEFAULT 0, - [Office Teams] [int] NOT NULL DEFAULT 0, - [Office Outlook Windows] [int] NOT NULL DEFAULT 0, - [Office Word Windows] [int] NOT NULL DEFAULT 0, - [Office Excel Windows] [int] NOT NULL DEFAULT 0, - [Office Powerpoint Windows] [int] NOT NULL DEFAULT 0, - [Office Onenote Windows] [int] NOT NULL DEFAULT 0, - [Office Teams Windows] [int] NOT NULL DEFAULT 0, - [Office Outlook Mac] [int] NOT NULL DEFAULT 0, - [Office Word Mac] [int] NOT NULL DEFAULT 0, - [Office Excel Mac] [int] NOT NULL DEFAULT 0, - [Office Powerpoint Mac] [int] NOT NULL DEFAULT 0, - [Office Onenote Mac] [int] NOT NULL DEFAULT 0, - [Office Teams Mac] [int] NOT NULL DEFAULT 0, - [Office Outlook Mobile] [int] NOT NULL DEFAULT 0, - [Office Word Mobile] [int] NOT NULL DEFAULT 0, - [Office Excel Mobile] [int] NOT NULL DEFAULT 0, - [Office Powerpoint Mobile] [int] NOT NULL DEFAULT 0, - [Office Onenote Mobile] [int] NOT NULL DEFAULT 0, - [Office Teams Mobile] [int] NOT NULL DEFAULT 0, - [Office Outlook Web] [int] NOT NULL DEFAULT 0, - [Office Word Web] [int] NOT NULL DEFAULT 0, - [Office Excel Web] [int] NOT NULL DEFAULT 0, - [Office Powerpoint Web] [int] NOT NULL DEFAULT 0, - [Office Onenote Web] [int] NOT NULL DEFAULT 0, - [Office Teams Web] [int] NOT NULL DEFAULT 0, - -- Yammer devices - [Yammer Platform Count] [int] NOT NULL DEFAULT 0, - [Yammer Used Web] [int] NOT NULL DEFAULT 0, - [Yammer Used Mobile] [int] NOT NULL DEFAULT 0, - [Yammer Used Others] [int] NOT NULL DEFAULT 0, - [Yammer Used WinPhone] [bit] NOT NULL DEFAULT 0, - [Yammer Used Android] [bit] NOT NULL DEFAULT 0, - [Yammer Used iPad] [bit] NOT NULL DEFAULT 0, - [Yammer Used iPhone] [bit] NOT NULL DEFAULT 0 + CREATE TABLE #UsageStaging + ( + "user_id" INT NOT NULL, + "date" DATETIME NOT NULL, + "Teams Used Web" INT NOT NULL DEFAULT 0, + "Teams Used Mac" INT NOT NULL DEFAULT 0, + "Teams Used Windows" INT NOT NULL DEFAULT 0, + "Teams Used Linux" INT NOT NULL DEFAULT 0, + "Teams Used Chrome OS" INT NOT NULL DEFAULT 0, + "Teams Used Mobile" INT NOT NULL DEFAULT 0, + "Teams Used WinPhone" INT NOT NULL DEFAULT 0, + "Teams Used iOS" INT NOT NULL DEFAULT 0, + "Teams Used Android" INT NOT NULL DEFAULT 0, + "Office Windows" INT NOT NULL DEFAULT 0, + "Office Mac" INT NOT NULL DEFAULT 0, + "Office Mobile" INT NOT NULL DEFAULT 0, + "Office Web" INT NOT NULL DEFAULT 0, + "Office Outlook" INT NOT NULL DEFAULT 0, + "Office Word" INT NOT NULL DEFAULT 0, + "Office Excel" INT NOT NULL DEFAULT 0, + "Office Powerpoint" INT NOT NULL DEFAULT 0, + "Office Onenote" INT NOT NULL DEFAULT 0, + "Office Teams" INT NOT NULL DEFAULT 0, + "Office Outlook Windows" INT NOT NULL DEFAULT 0, + "Office Word Windows" INT NOT NULL DEFAULT 0, + "Office Excel Windows" INT NOT NULL DEFAULT 0, + "Office Powerpoint Windows" INT NOT NULL DEFAULT 0, + "Office Onenote Windows" INT NOT NULL DEFAULT 0, + "Office Teams Windows" INT NOT NULL DEFAULT 0, + "Office Outlook Mac" INT NOT NULL DEFAULT 0, + "Office Word Mac" INT NOT NULL DEFAULT 0, + "Office Excel Mac" INT NOT NULL DEFAULT 0, + "Office Powerpoint Mac" INT NOT NULL DEFAULT 0, + "Office Onenote Mac" INT NOT NULL DEFAULT 0, + "Office Teams Mac" INT NOT NULL DEFAULT 0, + "Office Outlook Mobile" INT NOT NULL DEFAULT 0, + "Office Word Mobile" INT NOT NULL DEFAULT 0, + "Office Excel Mobile" INT NOT NULL DEFAULT 0, + "Office Powerpoint Mobile" INT NOT NULL DEFAULT 0, + "Office Onenote Mobile" INT NOT NULL DEFAULT 0, + "Office Teams Mobile" INT NOT NULL DEFAULT 0, + "Office Outlook Web" INT NOT NULL DEFAULT 0, + "Office Word Web" INT NOT NULL DEFAULT 0, + "Office Excel Web" INT NOT NULL DEFAULT 0, + "Office Powerpoint Web" INT NOT NULL DEFAULT 0, + "Office Onenote Web" INT NOT NULL DEFAULT 0, + "Office Teams Web" INT NOT NULL DEFAULT 0, + "Yammer Platform Count" INT NOT NULL DEFAULT 0, + "Yammer Used Web" INT NOT NULL DEFAULT 0, + "Yammer Used Mobile" INT NOT NULL DEFAULT 0, + "Yammer Used Others" INT NOT NULL DEFAULT 0, + "Yammer Used WinPhone" BIT NOT NULL DEFAULT 0, + "Yammer Used Android" BIT NOT NULL DEFAULT 0, + "Yammer Used iPad" BIT NOT NULL DEFAULT 0, + "Yammer Used iPhone" BIT NOT NULL DEFAULT 0 ); - - EXECUTE [profiling].[usp_UpsertTeamsDevices] @Monday, @Sunday; - EXECUTE [profiling].[usp_UpsertM365Apps] @Monday, @Sunday; - EXECUTE [profiling].[usp_UpsertYammerDevices] @Monday, @Sunday; - - INSERT INTO [profiling].[UsageWeekly] ( - user_id, date, - [Teams Used Web], [Teams Used Mac], [Teams Used Windows], [Teams Used Linux], [Teams Used Chrome OS], - [Teams Used Mobile], [Teams Used WinPhone], [Teams Used iOS], [Teams Used Android], - [Office Windows], [Office Mac], [Office Mobile], [Office Web], - [Office Outlook], [Office Word], [Office Excel], [Office Powerpoint], [Office Onenote], [Office Teams], - [Office Outlook Windows], [Office Word Windows], [Office Excel Windows], [Office Powerpoint Windows], [Office Onenote Windows], [Office Teams Windows], - [Office Outlook Mac], [Office Word Mac], [Office Excel Mac], [Office Powerpoint Mac], [Office Onenote Mac], [Office Teams Mac], - [Office Outlook Mobile], [Office Word Mobile], [Office Excel Mobile], [Office Powerpoint Mobile], [Office Onenote Mobile], [Office Teams Mobile], - [Office Outlook Web], [Office Word Web], [Office Excel Web], [Office Powerpoint Web], [Office Onenote Web], [Office Teams Web], - [Yammer Platform Count], - [Yammer Used Web], [Yammer Used Mobile], [Yammer Used Others], - [Yammer Used WinPhone], [Yammer Used Android], [Yammer Used iPad], [Yammer Used iPhone] + EXECUTE profiling.usp_UpsertTeamsDevices @Monday, @Sunday; + EXECUTE profiling.usp_UpsertM365Apps @Monday, @Sunday; + EXECUTE profiling.usp_UpsertYammerDevices @Monday, @Sunday; + + INSERT INTO profiling.UsageWeekly + ( + "user_id", + "date", + "Teams Used Web", + "Teams Used Mac", + "Teams Used Windows", + "Teams Used Linux", + "Teams Used Chrome OS", + "Teams Used Mobile", + "Teams Used WinPhone", + "Teams Used iOS", + "Teams Used Android", + "Office Windows", + "Office Mac", + "Office Mobile", + "Office Web", + "Office Outlook", + "Office Word", + "Office Excel", + "Office Powerpoint", + "Office Onenote", + "Office Teams", + "Office Outlook Windows", + "Office Word Windows", + "Office Excel Windows", + "Office Powerpoint Windows", + "Office Onenote Windows", + "Office Teams Windows", + "Office Outlook Mac", + "Office Word Mac", + "Office Excel Mac", + "Office Powerpoint Mac", + "Office Onenote Mac", + "Office Teams Mac", + "Office Outlook Mobile", + "Office Word Mobile", + "Office Excel Mobile", + "Office Powerpoint Mobile", + "Office Onenote Mobile", + "Office Teams Mobile", + "Office Outlook Web", + "Office Word Web", + "Office Excel Web", + "Office Powerpoint Web", + "Office Onenote Web", + "Office Teams Web", + "Yammer Platform Count", + "Yammer Used Web", + "Yammer Used Mobile", + "Yammer Used Others", + "Yammer Used WinPhone", + "Yammer Used Android", + "Yammer Used iPad", + "Yammer Used iPhone" ) SELECT - user_id, - @Monday as date, - [Teams Used Web], [Teams Used Mac], [Teams Used Windows], [Teams Used Linux], [Teams Used Chrome OS], - [Teams Used Mobile], [Teams Used WinPhone], [Teams Used iOS], [Teams Used Android], - [Office Windows], [Office Mac], [Office Mobile], [Office Web], - [Office Outlook], [Office Word], [Office Excel], [Office Powerpoint], [Office Onenote], [Office Teams], - [Office Outlook Windows], [Office Word Windows], [Office Excel Windows], [Office Powerpoint Windows], [Office Onenote Windows], [Office Teams Windows], - [Office Outlook Mac], [Office Word Mac], [Office Excel Mac], [Office Powerpoint Mac], [Office Onenote Mac], [Office Teams Mac], - [Office Outlook Mobile], [Office Word Mobile], [Office Excel Mobile], [Office Powerpoint Mobile], [Office Onenote Mobile], [Office Teams Mobile], - [Office Outlook Web], [Office Word Web], [Office Excel Web], [Office Powerpoint Web], [Office Onenote Web], [Office Teams Web], - [Yammer Platform Count], - [Yammer Used Web], [Yammer Used Mobile], [Yammer Used Others], - [Yammer Used WinPhone], [Yammer Used Android], [Yammer Used iPad], [Yammer Used iPhone] + "user_id", + @Monday AS "date", + "Teams Used Web", + "Teams Used Mac", + "Teams Used Windows", + "Teams Used Linux", + "Teams Used Chrome OS", + "Teams Used Mobile", + "Teams Used WinPhone", + "Teams Used iOS", + "Teams Used Android", + "Office Windows", + "Office Mac", + "Office Mobile", + "Office Web", + "Office Outlook", + "Office Word", + "Office Excel", + "Office Powerpoint", + "Office Onenote", + "Office Teams", + "Office Outlook Windows", + "Office Word Windows", + "Office Excel Windows", + "Office Powerpoint Windows", + "Office Onenote Windows", + "Office Teams Windows", + "Office Outlook Mac", + "Office Word Mac", + "Office Excel Mac", + "Office Powerpoint Mac", + "Office Onenote Mac", + "Office Teams Mac", + "Office Outlook Mobile", + "Office Word Mobile", + "Office Excel Mobile", + "Office Powerpoint Mobile", + "Office Onenote Mobile", + "Office Teams Mobile", + "Office Outlook Web", + "Office Word Web", + "Office Excel Web", + "Office Powerpoint Web", + "Office Onenote Web", + "Office Teams Web", + "Yammer Platform Count", + "Yammer Used Web", + "Yammer Used Mobile", + "Yammer Used Others", + "Yammer Used WinPhone", + "Yammer Used Android", + "Yammer Used iPad", + "Yammer Used iPhone" FROM #UsageStaging; DROP TABLE #UsageStaging; END END TRY BEGIN CATCH - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'Catch [usp_CompileUsageWeek]: ' + ERROR_MESSAGE(); - + DECLARE @ErrorMessage NVARCHAR(4000); + SELECT @ErrorMessage = ERROR_MESSAGE(); + EXEC profiling.usp_Trace 'Catch [usp_CompileUsageWeek]: %s', @ErrorMessage; IF OBJECT_ID('tempdb..#UsageStaging') IS NOT NULL + BEGIN DROP TABLE #UsageStaging; - END CATCH -END + END + END CATCH; +END; GO -- =================================== -- Aggregates a week of analytics data -- =================================== -IF OBJECT_ID(N'profiling.usp_CompileActivityWeek') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileActivityWeek]; + +IF OBJECT_ID(N'profiling.usp_CompileActivityWeek') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileActivityWeek; +END GO -CREATE PROCEDURE [profiling].[usp_CompileActivityWeek] ( - @Monday DATE -- Start day of the week to aggregate -) AS +CREATE PROCEDURE profiling.usp_CompileActivityWeek +( + -- Start day of the week to aggregate + @Monday DATE +) +AS BEGIN - -- SET NOCOUNT ON added to prevent extra result sets from interfering with SELECT statements. SET NOCOUNT ON; BEGIN TRY - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'[usp_CompileActivityWeek] Starting: ' + CAST(@Monday AS NVARCHAR); + EXEC profiling.usp_Trace '[usp_CompileActivityWeek] Starting: %s', @Monday; + DECLARE @Sunday DATE = DATEADD(DAY, 6, @Monday), - @RowsDone INT = 0, -- Unpivoted data - @ColumnsDone INT = 0 + @RowsDone INT = 0, + @ColumnsDone INT = 0; -- Check if the data has already been aggregated - SELECT @ColumnsDone=COUNT([date]) - FROM [profiling].[ActivitiesWeeklyColumns] - WHERE [date] = @Monday + SELECT @ColumnsDone = COUNT("date") + FROM profiling.ActivitiesWeeklyColumns + WHERE "date" = @Monday; - SELECT @RowsDone=COUNT([MetricDate]) - FROM [profiling].[ActivitiesWeekly] - WHERE [MetricDate] = @Monday + SELECT @RowsDone = COUNT(MetricDate) + FROM profiling.ActivitiesWeekly + WHERE MetricDate = @Monday; - -- If the data has not been aggregated, do it IF @RowsDone = 0 OR @ColumnsDone = 0 BEGIN - CREATE TABLE #ActivitiesStaging ( - [user_id] BIGINT, - [date] DATE, - - [OneDrive Viewed/Edited] BIGINT DEFAULT 0, - [OneDrive Synced] BIGINT DEFAULT 0, - [OneDrive Shared Internally] BIGINT DEFAULT 0, - [OneDrive Shared Externally] BIGINT DEFAULT 0, - - [SPO Viewed/Edited] BIGINT DEFAULT 0, - [SPO Synced] BIGINT DEFAULT 0, - [SPO Shared Internally] BIGINT DEFAULT 0, - [SPO Shared Externally] BIGINT DEFAULT 0, - - [Emails Sent] BIGINT DEFAULT 0, - [Emails Received] BIGINT DEFAULT 0, - [Emails Read] BIGINT DEFAULT 0, - [Outlook Meetings Created] BIGINT DEFAULT 0, - [Outlook Meetings Interacted] BIGINT DEFAULT 0, - - [Teams Private Chats] BIGINT DEFAULT 0, - [Teams Team Chats] BIGINT DEFAULT 0, - [Teams Calls] BIGINT DEFAULT 0, - [Teams Meetings] BIGINT DEFAULT 0, - [Teams Meetings Attended] BIGINT DEFAULT 0, - [Teams Meetings Organized] BIGINT DEFAULT 0, - [Teams Adhoc Meetings Attended] BIGINT DEFAULT 0, - [Teams Adhoc Meetings Organized] BIGINT DEFAULT 0, - [Teams Scheduled Onetime Meetings Attended] BIGINT DEFAULT 0, - [Teams Scheduled Onetime Meetings Organized] BIGINT DEFAULT 0, - [Teams Scheduled Recurring Meetings Attended] BIGINT DEFAULT 0, - [Teams Scheduled Recurring Meetings Organized] BIGINT DEFAULT 0, - [Teams Audio Duration Seconds] BIGINT DEFAULT 0, - [Teams Video Duration Seconds] BIGINT DEFAULT 0, - [Teams Screenshare Duration Seconds] BIGINT DEFAULT 0, - [Teams Post Messages] BIGINT DEFAULT 0, - [Teams Reply Messages] BIGINT DEFAULT 0, - [Teams Urgent Messages] BIGINT DEFAULT 0, - - [Yammer Posted] BIGINT DEFAULT 0, - [Yammer Read] BIGINT DEFAULT 0, - [Yammer Liked] BIGINT DEFAULT 0 + CREATE TABLE #ActivitiesStaging + ( + "user_id" BIGINT, + "date" DATE, + "OneDrive Viewed/Edited" BIGINT DEFAULT 0, + "OneDrive Synced" BIGINT DEFAULT 0, + "OneDrive Shared Internally" BIGINT DEFAULT 0, + "OneDrive Shared Externally" BIGINT DEFAULT 0, + "SPO Viewed/Edited" BIGINT DEFAULT 0, + "SPO Synced" BIGINT DEFAULT 0, + "SPO Shared Internally" BIGINT DEFAULT 0, + "SPO Shared Externally" BIGINT DEFAULT 0, + "Emails Sent" BIGINT DEFAULT 0, + "Emails Received" BIGINT DEFAULT 0, + "Emails Read" BIGINT DEFAULT 0, + "Outlook Meetings Created" BIGINT DEFAULT 0, + "Outlook Meetings Interacted" BIGINT DEFAULT 0, + "Teams Private Chats" BIGINT DEFAULT 0, + "Teams Team Chats" BIGINT DEFAULT 0, + "Teams Calls" BIGINT DEFAULT 0, + "Teams Meetings" BIGINT DEFAULT 0, + "Teams Meetings Attended" BIGINT DEFAULT 0, + "Teams Meetings Organized" BIGINT DEFAULT 0, + "Teams Adhoc Meetings Attended" BIGINT DEFAULT 0, + "Teams Adhoc Meetings Organized" BIGINT DEFAULT 0, + "Teams Scheduled Onetime Meetings Attended" BIGINT DEFAULT 0, + "Teams Scheduled Onetime Meetings Organized" BIGINT DEFAULT 0, + "Teams Scheduled Recurring Meetings Attended" BIGINT DEFAULT 0, + "Teams Scheduled Recurring Meetings Organized" BIGINT DEFAULT 0, + "Teams Audio Duration Seconds" BIGINT DEFAULT 0, + "Teams Video Duration Seconds" BIGINT DEFAULT 0, + "Teams Screenshare Duration Seconds" BIGINT DEFAULT 0, + "Teams Post Messages" BIGINT DEFAULT 0, + "Teams Reply Messages" BIGINT DEFAULT 0, + "Teams Urgent Messages" BIGINT DEFAULT 0, + "Yammer Posted" BIGINT DEFAULT 0, + "Yammer Read" BIGINT DEFAULT 0, + "Yammer Liked" BIGINT DEFAULT 0, + "Copilot Chats" BIGINT DEFAULT 0, + "Copilot Meetings" BIGINT DEFAULT 0, + "Copilot Files" BIGINT DEFAULT 0, + "Copilot App Assist365" BIGINT DEFAULT 0, + "Copilot App Bing" BIGINT DEFAULT 0, + "Copilot App BashTool" BIGINT DEFAULT 0, + "Copilot App DevUI" BIGINT DEFAULT 0, + "Copilot App Excel" BIGINT DEFAULT 0, + "Copilot App Loop" BIGINT DEFAULT 0, + "Copilot App M365AdminCenter" BIGINT DEFAULT 0, + "Copilot App M365App" BIGINT DEFAULT 0, + "Copilot App Office" BIGINT DEFAULT 0, + "Copilot App OneNote" BIGINT DEFAULT 0, + "Copilot App Outlook" BIGINT DEFAULT 0, + "Copilot App Planner" BIGINT DEFAULT 0, + "Copilot App PowerPoint" BIGINT DEFAULT 0, + "Copilot App SharePoint" BIGINT DEFAULT 0, + "Copilot App Stream" BIGINT DEFAULT 0, + "Copilot App Teams" BIGINT DEFAULT 0, + "Copilot App VivaCopilot" BIGINT DEFAULT 0, + "Copilot App VivaEngage" BIGINT DEFAULT 0, + "Copilot App VivaGoals" BIGINT DEFAULT 0, + "Copilot App Whiteboard" BIGINT DEFAULT 0, + "Copilot App Word" BIGINT DEFAULT 0 ); -- Insert only the activities between the dates - EXECUTE [profiling].[usp_UpsertTeams] @Monday, @Sunday; - EXECUTE [profiling].[usp_UpsertOneDrive] @Monday, @Sunday; - EXECUTE [profiling].[usp_UpsertSharePoint] @Monday, @Sunday; - EXECUTE [profiling].[usp_UpsertOutlook] @Monday, @Sunday; - EXECUTE [profiling].[usp_UpsertYammer] @Monday, @Sunday; + EXECUTE profiling.usp_UpsertTeams @Monday, @Sunday; + EXECUTE profiling.usp_UpsertOneDrive @Monday, @Sunday; + EXECUTE profiling.usp_UpsertSharePoint @Monday, @Sunday; + EXECUTE profiling.usp_UpsertOutlook @Monday, @Sunday; + EXECUTE profiling.usp_UpsertYammer @Monday, @Sunday; + EXECUTE profiling.usp_UpsertCopilot @Monday, @Sunday; IF @ColumnsDone = 0 - EXECUTE [profiling].[usp_CompileWeekActivityColumns] @Monday; + BEGIN + EXECUTE profiling.usp_CompileWeekActivityColumns @Monday; + END + IF @RowsDone = 0 - EXECUTE [profiling].[usp_CompileWeekActivityRows] @Monday; - + BEGIN + EXECUTE profiling.usp_CompileWeekActivityRows @Monday; + END + DROP TABLE #ActivitiesStaging; END END TRY BEGIN CATCH - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'[usp_CompileActivityWeek] Catch: ' + ERROR_MESSAGE(); - + DECLARE @ErrorMessage NVARCHAR(4000); + SELECT @ErrorMessage = ERROR_MESSAGE(); + EXEC profiling.usp_Trace '[usp_CompileActivityWeek] Catch: %s', @ErrorMessage; IF OBJECT_ID('tempdb..#ActivitiesStaging') IS NOT NULL + BEGIN DROP TABLE #ActivitiesStaging; - END CATCH -END + END + END CATCH; +END; GO -- ============================================= -- Aggregates all possible weekly analytics data -- ============================================= -IF OBJECT_ID(N'profiling.usp_CompileWeekly') IS NOT NULL DROP PROCEDURE [profiling].[usp_CompileWeekly]; + +IF OBJECT_ID(N'profiling.usp_CompileWeekly') IS NOT NULL +BEGIN + DROP PROCEDURE profiling.usp_CompileWeekly; +END GO -CREATE PROCEDURE [profiling].[usp_CompileWeekly] ( +CREATE PROCEDURE profiling.usp_CompileWeekly +( @WeeksToKeep INT, @All INT = 0 ) @@ -1354,45 +2777,62 @@ BEGIN -- Today is the first day that there should be data, which usually is Today-4 days DECLARE @Today DATE = DATEADD(DAY, -4, GETDATE()); -- Day when aggregation should stop - DECLARE @ThisWeeksMonday DATE = [profiling].[udf_GetMonday](@Today); + DECLARE @ThisWeeksMonday DATE = profiling.udf_GetMonday (@Today); -- When data aggregation starts DECLARE @RetentionDate DATE = DATEADD(WEEK, -1 * @WeeksToKeep, @ThisWeeksMonday); -- Get last aggregated date in the table, it will be a Monday - DECLARE - @LastDateInTables DATE - ,@MaxActitiesWeekly DATE - ,@MaxActitiesWeeklyColumns DATE - ,@MaxUsageWeekly DATE; - SELECT @MaxActitiesWeekly = MAX(MetricDate) FROM [profiling].[ActivitiesWeekly]; - SELECT @MaxActitiesWeeklyColumns = MAX([date]) FROM [profiling].[ActivitiesWeeklyColumns]; - SELECT @MaxUsageWeekly = MAX([date]) FROM [profiling].[UsageWeekly]; - SELECT @LastDateInTables = MIN([date]) FROM (VALUES (@MaxActitiesWeekly), (@MaxActitiesWeeklyColumns), (@MaxUsageWeekly)) AS x([date]) + DECLARE @LastDateInTables DATE; + + -- Dates in the aggregated weekly tables + WITH + dates AS ( + SELECT MAX(MetricDate) AS "date" FROM profiling.ActivitiesWeekly + UNION + SELECT MAX("date") FROM profiling.ActivitiesWeeklyColumns + UNION + SELECT MAX("date") FROM profiling.UsageWeekly + ) + SELECT @LastDateInTables = MIN("date") + FROM dates; IF @LastDateInTables IS NULL OR @All = 1 - -- Start from the retention date - SET @LastDateInTables = @RetentionDate; + BEGIN + -- Start from the retention date + SELECT @LastDateInTables = @RetentionDate; + END -- Week by week, aggregate the data DECLARE @Monday DATE = DATEADD(DAY, 7, @LastDateInTables); - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'Weekly aggregation requested, from ' + CAST(@Monday as NVARCHAR) + ' to possibly ' + CAST(@ThisWeeksMonday as NVARCHAR); + + EXEC profiling.usp_Trace + N'Weekly aggregation requested. First: %s. Last: %s', + @Monday, + @ThisWeeksMonday; + WHILE @ThisWeeksMonday > @Monday BEGIN - DECLARE @Sunday DATE = DATEADD(DAY, 6, @Monday); - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'Week from ' + CAST(@Monday AS NVARCHAR) + ' to ' + CAST(@Sunday AS NVARCHAR); - EXECUTE [profiling].[usp_CompileActivityWeek] @Monday; - EXECUTE [profiling].[usp_CompileUsageWeek] @Monday; - SET @Monday = DATEADD(DAY, 7, @Monday); + DECLARE @Sunday DATE = DATEADD(DAY, 6, @Monday); + + EXEC profiling.usp_Trace 'Week from %s to %s', @Monday, @Sunday; + + EXECUTE profiling.usp_CompileActivityWeek @Monday; + EXECUTE profiling.usp_CompileUsageWeek @Monday; + + SELECT @Monday = DATEADD(DAY, 7, @Monday); END -- Cleanup. Remove data in the tables before the retention date - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'Starting cleanup. Retention date: ' + CAST(@RetentionDate AS NVARCHAR); - DELETE FROM [profiling].[ActivitiesWeekly] WHERE [MetricDate] < @RetentionDate; - DELETE FROM [profiling].[ActivitiesWeeklyColumns] WHERE [date] < @RetentionDate; - DELETE FROM [profiling].[UsageWeekly] WHERE [date] < @RetentionDate; - INSERT INTO [profiling].[TraceLogs] ([Datetime], [Message]) - SELECT GETDATE(), N'Aggregation finished'; -END + EXEC profiling.usp_Trace 'Starting cleanup. Retention date: %s', @RetentionDate; + + DELETE FROM profiling.ActivitiesWeekly + WHERE MetricDate < @RetentionDate; + + DELETE FROM profiling.ActivitiesWeeklyColumns + WHERE "date" < @RetentionDate; + + DELETE FROM profiling.UsageWeekly + WHERE "date" < @RetentionDate; + + EXEC profiling.usp_Trace 'Aggregation finished'; +END; GO diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/SqlExtentions/readme.md b/src/AnalyticsEngine/App.ControlPanel.Engine/SqlExtentions/readme.md index 952944d..b8545f2 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/SqlExtentions/readme.md +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/SqlExtentions/readme.md @@ -8,5 +8,5 @@ This is useful for adding custom SQL scripts to the installer, like the profilin This is done as part of the standard database upgrade process, either via the UI or command-line execution (usually by another instance of the installer actually executing the upgrade). The scripts are run after the database schema has been updated with Entity Framework migrations. -Note: SQL server doesn't normally support "```GO```" statements in stored procedures ("GO" is a SQL Server Management Studio thing to seperate scripts in a single file), so you should avoid using them in your scripts here. +Note: SQL server doesn't normally support "```GO```" statements in stored procedures ("GO" is a SQL Server Management Studio thing to separate scripts in a single file), so you should avoid using them in your scripts here. You *can* use the "GO" statement and the installer will split each segment by "GO" and then execute each segment (bit of a hack), but you should put the scripts in separate files or remove "GO" from any script here. diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/DatabaseUpgrader.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/DatabaseUpgrader.cs index d0313d5..602c4a6 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/DatabaseUpgrader.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/DatabaseUpgrader.cs @@ -1,6 +1,6 @@ using App.ControlPanel.Engine.Models; -using Common.DataUtils; using Common.Entities; +using DataUtils; using System; using System.Collections.Generic; using System.Linq; @@ -43,7 +43,7 @@ public static void CheckDbUpgraded(DatabaseUpgradeInfo initInfo, Action } // Run custom SQL scripts - var rr = new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()); + var rr = new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()); try { using (var context = new AnalyticsEntitiesContext(initInfo.ConnectionString, true, false)) @@ -53,7 +53,7 @@ public static void CheckDbUpgraded(DatabaseUpgradeInfo initInfo, Action foreach (var scriptName in sqlScriptNames) { log?.Invoke($"--Running script '{scriptName}'..."); - var script = rr.ReadResourceStringFromExecutingAssembly(scriptName); + var script = rr.ReadResourceString(scriptName); var statements = SplitSqlStatements(script); foreach (var statement in statements) diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/SqlInstallerTasks.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/SqlInstallerTasks.cs index 4445e5c..dbe226e 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/SqlInstallerTasks.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/SqlInstallerTasks.cs @@ -1,6 +1,6 @@ using App.ControlPanel.Engine.Entities; using App.ControlPanel.Engine.Models; -using Common.DataUtils; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/ZipFileTasks.cs b/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/ZipFileTasks.cs index 7ef1836..2b2cb12 100644 --- a/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/ZipFileTasks.cs +++ b/src/AnalyticsEngine/App.ControlPanel.Engine/Utils/ZipFileTasks.cs @@ -1,5 +1,5 @@ using App.ControlPanel.Engine.Entities; -using Common.DataUtils; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.IO; diff --git a/src/AnalyticsEngine/App.ControlPanel/Controls/TagsEditor.cs b/src/AnalyticsEngine/App.ControlPanel/Controls/TagsEditor.cs index e063718..1904b5e 100644 --- a/src/AnalyticsEngine/App.ControlPanel/Controls/TagsEditor.cs +++ b/src/AnalyticsEngine/App.ControlPanel/Controls/TagsEditor.cs @@ -1,5 +1,4 @@ -using Azure.ResourceManager.Resources.Models; -using Common.Entities.Installer; +using Common.Entities.Installer; using System; using System.Collections.Generic; using System.Windows.Forms; @@ -94,7 +93,7 @@ private void btnSaveNew_Click(object sender, EventArgs e) lstTags.Items.Add(new TagLVI(new AzTag(name, val))); txtTagName.Text = string.Empty; - txtTagVal.Text = string.Empty; + txtTagVal.Text = string.Empty; UpdateDisplayControls(View.List); } diff --git a/src/AnalyticsEngine/App.ControlPanel/Frames/InstallSPOSitesControl.cs b/src/AnalyticsEngine/App.ControlPanel/Frames/InstallSPOSitesControl.cs index 92b4732..ddc641a 100644 --- a/src/AnalyticsEngine/App.ControlPanel/Frames/InstallSPOSitesControl.cs +++ b/src/AnalyticsEngine/App.ControlPanel/Frames/InstallSPOSitesControl.cs @@ -1,7 +1,7 @@ using App.ControlPanel.Engine; using App.ControlPanel.Engine.Entities; using App.ControlPanel.Engine.Models; -using Common.DataUtils; +using DataUtils; using System; using System.Configuration; using System.Windows.Forms; @@ -146,7 +146,7 @@ public void ConfigureUI(SolutionInstallConfig config) azureBaseConfigControl1.ResourceGroup = config.ResourceGroupName; azureBaseConfigControl1.AzureSubscription = config.Subscription; azureBaseConfigControl1.AzureLocationString = config.AzureLocation; - azureBaseConfigControl1.EnvironmentType = config.EnvironmentType; + azureBaseConfigControl1.EnvironmentType = config.EnvironmentType; azureBaseConfigControl1.Tags = config.Tags; azureStorageConfigControl1.SQLDb = config.SQLServerDatabaseName; diff --git a/src/AnalyticsEngine/App.ControlPanel/Frames/InstallWizardPages/AzureBaseConfigControl.cs b/src/AnalyticsEngine/App.ControlPanel/Frames/InstallWizardPages/AzureBaseConfigControl.cs index 86e0a92..eb88633 100644 --- a/src/AnalyticsEngine/App.ControlPanel/Frames/InstallWizardPages/AzureBaseConfigControl.cs +++ b/src/AnalyticsEngine/App.ControlPanel/Frames/InstallWizardPages/AzureBaseConfigControl.cs @@ -3,8 +3,8 @@ using App.ControlPanel.Engine.Models; using Azure.Identity; using Azure.ResourceManager; -using Common.DataUtils; using Common.Entities.Installer; +using DataUtils; using System; using System.Collections.Generic; using System.Linq; @@ -158,16 +158,16 @@ public EnvironmentTypeEnum EnvironmentType } } - public List Tags - { - get - { + public List Tags + { + get + { return tagsEditor.Tags; - } - set + } + set { tagsEditor.Tags = value; - } + } } #endregion diff --git a/src/AnalyticsEngine/App.ControlPanel/InstallerPreferences.cs b/src/AnalyticsEngine/App.ControlPanel/InstallerPreferences.cs index 8551de4..364e1be 100644 --- a/src/AnalyticsEngine/App.ControlPanel/InstallerPreferences.cs +++ b/src/AnalyticsEngine/App.ControlPanel/InstallerPreferences.cs @@ -1,6 +1,6 @@ using App.ControlPanel.Engine.Entities; using App.ControlPanel.Engine.Models; -using Common.DataUtils; +using DataUtils; namespace App.ControlPanel { diff --git a/src/AnalyticsEngine/App.ControlPanel/MainForm.cs b/src/AnalyticsEngine/App.ControlPanel/MainForm.cs index cbaa6d1..8a9bafc 100644 --- a/src/AnalyticsEngine/App.ControlPanel/MainForm.cs +++ b/src/AnalyticsEngine/App.ControlPanel/MainForm.cs @@ -1,6 +1,6 @@ using App.ControlPanel.Engine; using App.ControlPanel.Engine.Models; -using Common.DataUtils; +using DataUtils; using System; using System.Windows.Forms; diff --git a/src/AnalyticsEngine/App.ControlPanel/NewSaltForm.cs b/src/AnalyticsEngine/App.ControlPanel/NewSaltForm.cs index d681496..c4c2938 100644 --- a/src/AnalyticsEngine/App.ControlPanel/NewSaltForm.cs +++ b/src/AnalyticsEngine/App.ControlPanel/NewSaltForm.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System; using System.Windows.Forms; diff --git a/src/AnalyticsEngine/App.ControlPanel/Program.cs b/src/AnalyticsEngine/App.ControlPanel/Program.cs index 42eddcd..0b5070d 100644 --- a/src/AnalyticsEngine/App.ControlPanel/Program.cs +++ b/src/AnalyticsEngine/App.ControlPanel/Program.cs @@ -1,7 +1,7 @@ using App.ControlPanel.Engine; using App.ControlPanel.Engine.Models; -using Common.DataUtils; using Common.Entities; +using DataUtils; using System; using System.Diagnostics; using System.IO; diff --git a/src/AnalyticsEngine/App.ControlPanel/SolutionInstallerLogger.cs b/src/AnalyticsEngine/App.ControlPanel/SolutionInstallerLogger.cs index 6241ad2..fd05d06 100644 --- a/src/AnalyticsEngine/App.ControlPanel/SolutionInstallerLogger.cs +++ b/src/AnalyticsEngine/App.ControlPanel/SolutionInstallerLogger.cs @@ -1,6 +1,6 @@ using App.ControlPanel.Engine.Models; using App.ControlPanel.Frames; -using Common.DataUtils; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/Abstract.cs b/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/Abstract.cs index 75725b9..ad20574 100644 --- a/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/Abstract.cs +++ b/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/Abstract.cs @@ -1,9 +1,6 @@ -using Azure; -using Azure.Core; +using Azure.Core; using Azure.ResourceManager.Resources; -using Azure.ResourceManager.Resources.Models; using Microsoft.Extensions.Logging; -using System; using System.Collections.Generic; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/GenericArmSingleResourceInstallTask.cs b/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/GenericArmSingleResourceInstallTask.cs index d561121..27af2d1 100644 --- a/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/GenericArmSingleResourceInstallTask.cs +++ b/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/GenericArmSingleResourceInstallTask.cs @@ -15,7 +15,7 @@ namespace CloudInstallEngine.Azure.InstallTasks /// public abstract class GenericArmSingleResourceInstallTask : InstallTaskInAzResourceGroup { - public GenericArmSingleResourceInstallTask(TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) + public GenericArmSingleResourceInstallTask(TaskConfig config, ILogger logger, AzureLocation azureLocation, Dictionary tags) : base(config, logger, azureLocation, tags) { } @@ -66,7 +66,7 @@ public async Task CreateGenericAzResourceAndEnsureTags(strin if (string.IsNullOrEmpty(type)) throw new ArgumentException($"'{nameof(type)}' cannot be null or empty.", nameof(type)); if (newCreationParams is null) throw new ArgumentNullException(nameof(newCreationParams)); if (string.IsNullOrEmpty(templateJson)) throw new ArgumentException($"'{nameof(templateJson)}' cannot be null or empty.", nameof(templateJson)); - + var resource = GetGenericAzResource(name, type); if (resource == null) diff --git a/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/KeyVaultTasks.cs b/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/KeyVaultTasks.cs index 2f55dc3..3ddded8 100644 --- a/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/KeyVaultTasks.cs +++ b/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/KeyVaultTasks.cs @@ -101,7 +101,9 @@ protected async Task AddPolicyForConfiguredRuntimeAccount(KeyVaultResource vault // Only support adding accounts from same tenant as KV if (tenantId != vaultResource.Data.Properties.TenantId) { - throw new InstallException($"Application ID {clientId} does not exist in the Key Vault tenant ID {vaultResource.Data.Properties.TenantId}"); + _logger.LogError($"Key Vault permissions configuration error: Entra ID application ID {clientId} does not exist in the Key Vault tenant ID {vaultResource.Data.Properties.TenantId}"); + _logger.LogError("Continuing anyway, but your Key Vault is NOT configured due to an unsupported setup - both Office 365 and Azure should be in the same Entra ID tenant"); + return; } _logger.LogInformation($"Adding Azure AD application with client ID '{clientId}' to key vault {vaultResource.Data.Name} for secret read & list; certificate read"); diff --git a/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/SqlServerTask.cs b/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/SqlServerTask.cs index ab692e6..8c3e704 100644 --- a/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/SqlServerTask.cs +++ b/src/AnalyticsEngine/Common/CloudInstallEngine/Azure/InstallTasks/SqlServerTask.cs @@ -2,7 +2,6 @@ using Azure.Core; using Azure.ResourceManager.Sql; using Microsoft.Extensions.Logging; -using System; using System.Collections.Generic; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/Common/CloudInstallEngine/InstallJobInContainer.cs b/src/AnalyticsEngine/Common/CloudInstallEngine/InstallJobInContainer.cs index fa4393e..4ac059c 100644 --- a/src/AnalyticsEngine/Common/CloudInstallEngine/InstallJobInContainer.cs +++ b/src/AnalyticsEngine/Common/CloudInstallEngine/InstallJobInContainer.cs @@ -37,9 +37,10 @@ public virtual async Task Install(object previousRunResult) { previousRunResult = await ProcessTask(thisTask, previousRunResult); } - catch (Exception) + catch (Exception ex) { Logger.LogError($"Unexpected error on stage {thisTask.TaskName}:"); + Logger.LogError(ex, ex.Message); throw; // Error will be logged by parent } diff --git a/src/AnalyticsEngine/Common/CloudInstallEngine/InstallTask.cs b/src/AnalyticsEngine/Common/CloudInstallEngine/InstallTask.cs index c4379ba..73eddd2 100644 --- a/src/AnalyticsEngine/Common/CloudInstallEngine/InstallTask.cs +++ b/src/AnalyticsEngine/Common/CloudInstallEngine/InstallTask.cs @@ -1,6 +1,6 @@ -using Azure.ResourceManager.Resources.Models; +using Azure; using Azure.ResourceManager.Resources; -using Azure; +using Azure.ResourceManager.Resources.Models; using CloudInstallEngine.Models; using Microsoft.Extensions.Logging; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/Common/DataUtils/AnalyticsLogger.cs b/src/AnalyticsEngine/Common/DataUtils/AnalyticsLogger.cs index 6e45e4f..8b0254b 100644 --- a/src/AnalyticsEngine/Common/DataUtils/AnalyticsLogger.cs +++ b/src/AnalyticsEngine/Common/DataUtils/AnalyticsLogger.cs @@ -3,13 +3,13 @@ using System; using System.Collections.Generic; -namespace Common.DataUtils +namespace DataUtils { public abstract class BaseAnalyticsLogger : ILogger { public bool IsEnabled(LogLevel logLevel) { - return (logLevel == LogLevel.Information || logLevel == LogLevel.Warning || logLevel == LogLevel.Error || logLevel == LogLevel.Critical); + return logLevel == LogLevel.Information || logLevel == LogLevel.Warning || logLevel == LogLevel.Error || logLevel == LogLevel.Critical; } public IDisposable BeginScope(TState state) diff --git a/src/AnalyticsEngine/Common/DataUtils/AuthHelper.cs b/src/AnalyticsEngine/Common/DataUtils/AuthHelper.cs index 09dad42..5b71326 100644 --- a/src/AnalyticsEngine/Common/DataUtils/AuthHelper.cs +++ b/src/AnalyticsEngine/Common/DataUtils/AuthHelper.cs @@ -6,7 +6,7 @@ using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; -namespace Common.DataUtils +namespace DataUtils { public class AuthHelper { diff --git a/src/AnalyticsEngine/Common/DataUtils/CognitiveExtensions.cs b/src/AnalyticsEngine/Common/DataUtils/CognitiveExtensions.cs index a6ae7cb..3596690 100644 --- a/src/AnalyticsEngine/Common/DataUtils/CognitiveExtensions.cs +++ b/src/AnalyticsEngine/Common/DataUtils/CognitiveExtensions.cs @@ -6,7 +6,7 @@ using System.Linq; using System.Threading.Tasks; -namespace Common.DataUtils +namespace DataUtils { public class TextAnalysisSample where T : class { @@ -52,7 +52,7 @@ public static async Task>> GetCognitiveDataStats(t { // Break down API calls into chunks of 10 max and compile results var listProcessorLang = new ParallelCallsForSingleReturnListHander, DetectLanguageResult>(); - detectedLanguages = await listProcessorLang.CallAndCompileToSingleList(inputData, async (List> chunk) => + detectedLanguages = await listProcessorLang.CallAndCompileToSingleList(inputData, async (chunk) => { var languageBatchInput = new List(); foreach (var d in inputData) @@ -127,7 +127,7 @@ public static async Task>> GetCognitiveDataStats(t try { // Break down API calls into chunks of 10 max and compile results - allAnalyzeSentimentResults = await listProcessorSentiment.CallAndCompileToSingleList(sentimentBatchInput, async (List chunk) => + allAnalyzeSentimentResults = await listProcessorSentiment.CallAndCompileToSingleList(sentimentBatchInput, async (chunk) => { var result = await client.AnalyzeSentimentBatchAsync(chunk); return result.Value.ToList(); diff --git a/src/AnalyticsEngine/Common/DataUtils/CommonExceptionHandler.cs b/src/AnalyticsEngine/Common/DataUtils/CommonExceptionHandler.cs index d2b4d40..70800df 100644 --- a/src/AnalyticsEngine/Common/DataUtils/CommonExceptionHandler.cs +++ b/src/AnalyticsEngine/Common/DataUtils/CommonExceptionHandler.cs @@ -1,6 +1,6 @@ using System; -namespace Common.DataUtils +namespace DataUtils { public class CommonExceptionHandler { diff --git a/src/AnalyticsEngine/Common/DataUtils/ConcurrentLookupDbIdsCache.cs b/src/AnalyticsEngine/Common/DataUtils/ConcurrentLookupDbIdsCache.cs index 3162f85..19947ed 100644 --- a/src/AnalyticsEngine/Common/DataUtils/ConcurrentLookupDbIdsCache.cs +++ b/src/AnalyticsEngine/Common/DataUtils/ConcurrentLookupDbIdsCache.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Concurrent; -namespace Common.DataUtils +namespace DataUtils { /// /// Cache IDs for a 1 or more database record types. Threadsafe. diff --git a/src/AnalyticsEngine/Common/DataUtils/ConsoleApp.cs b/src/AnalyticsEngine/Common/DataUtils/ConsoleApp.cs index cfb61aa..5479829 100644 --- a/src/AnalyticsEngine/Common/DataUtils/ConsoleApp.cs +++ b/src/AnalyticsEngine/Common/DataUtils/ConsoleApp.cs @@ -2,7 +2,7 @@ using System; using System.Configuration; -namespace Common.DataUtils +namespace DataUtils { /// /// Provides utilities for console apps. diff --git a/src/AnalyticsEngine/Common/DataUtils/Http/AutoThrottleHttpClient.cs b/src/AnalyticsEngine/Common/DataUtils/Http/AutoThrottleHttpClient.cs index 8c665f7..2dc704a 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Http/AutoThrottleHttpClient.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Http/AutoThrottleHttpClient.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Common.DataUtils.Http +namespace DataUtils.Http { public class AutoThrottleHttpClient : HttpClient { diff --git a/src/AnalyticsEngine/Common/DataUtils/Http/ConfidentialClientApplicationThrottledHttpClient.cs b/src/AnalyticsEngine/Common/DataUtils/Http/ConfidentialClientApplicationThrottledHttpClient.cs index 0112c7c..d4aad9e 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Http/ConfidentialClientApplicationThrottledHttpClient.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Http/ConfidentialClientApplicationThrottledHttpClient.cs @@ -6,7 +6,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Common.DataUtils.Http +namespace DataUtils.Http { /// /// HttpClient that can handle HTTP 429s automatically diff --git a/src/AnalyticsEngine/Common/DataUtils/Http/HttpClientExtensions.cs b/src/AnalyticsEngine/Common/DataUtils/Http/HttpClientExtensions.cs index ab9b297..819e19c 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Http/HttpClientExtensions.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Http/HttpClientExtensions.cs @@ -4,7 +4,7 @@ using System.Net.Http.Headers; using System.Threading.Tasks; -namespace Common.DataUtils.Http +namespace DataUtils.Http { public static class HttpClientExtensions { diff --git a/src/AnalyticsEngine/Common/DataUtils/Http/OAuthContext.cs b/src/AnalyticsEngine/Common/DataUtils/Http/OAuthContext.cs index e14bdec..d253c6a 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Http/OAuthContext.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Http/OAuthContext.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Common.DataUtils.Http +namespace DataUtils.Http { /// /// Provides OAuth tokens for application-identity (client credentials) diff --git a/src/AnalyticsEngine/Common/DataUtils/IUrlObjectResolver.cs b/src/AnalyticsEngine/Common/DataUtils/IUrlObjectResolver.cs index 05220a1..e4d3ce0 100644 --- a/src/AnalyticsEngine/Common/DataUtils/IUrlObjectResolver.cs +++ b/src/AnalyticsEngine/Common/DataUtils/IUrlObjectResolver.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace Common.DataUtils +namespace DataUtils { public static class IUrlObjectResolver { diff --git a/src/AnalyticsEngine/Common/DataUtils/JobTimer.cs b/src/AnalyticsEngine/Common/DataUtils/JobTimer.cs index 3dc4665..107ba2a 100644 --- a/src/AnalyticsEngine/Common/DataUtils/JobTimer.cs +++ b/src/AnalyticsEngine/Common/DataUtils/JobTimer.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using static Common.DataUtils.AnalyticsLogger; +using static DataUtils.AnalyticsLogger; -namespace Common.DataUtils +namespace DataUtils { /// /// For tracking the time something takes diff --git a/src/AnalyticsEngine/Common/DataUtils/ListBatchProcessor.cs b/src/AnalyticsEngine/Common/DataUtils/ListBatchProcessor.cs index 611fc47..f03b5d9 100644 --- a/src/AnalyticsEngine/Common/DataUtils/ListBatchProcessor.cs +++ b/src/AnalyticsEngine/Common/DataUtils/ListBatchProcessor.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Threading.Tasks; -namespace Common.DataUtils +namespace DataUtils { /// /// Process list items into batches. diff --git a/src/AnalyticsEngine/Common/DataUtils/ObjectByIdCache.cs b/src/AnalyticsEngine/Common/DataUtils/ObjectByIdCache.cs index 86ac2bc..47cca9b 100644 --- a/src/AnalyticsEngine/Common/DataUtils/ObjectByIdCache.cs +++ b/src/AnalyticsEngine/Common/DataUtils/ObjectByIdCache.cs @@ -3,7 +3,7 @@ using System.Globalization; using System.Threading.Tasks; -namespace Common.DataUtils +namespace DataUtils { public abstract class ObjectByIdCache where T : class { diff --git a/src/AnalyticsEngine/Common/DataUtils/ParallelCallsForSingleReturnListHander.cs b/src/AnalyticsEngine/Common/DataUtils/ParallelCallsForSingleReturnListHander.cs index d75998f..52b1633 100644 --- a/src/AnalyticsEngine/Common/DataUtils/ParallelCallsForSingleReturnListHander.cs +++ b/src/AnalyticsEngine/Common/DataUtils/ParallelCallsForSingleReturnListHander.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Threading.Tasks; -namespace Common.DataUtils +namespace DataUtils { /// /// When you have a big list of items to process in parallel chunks, this class can help. diff --git a/src/AnalyticsEngine/Common/DataUtils/ParallelListProcessor.cs b/src/AnalyticsEngine/Common/DataUtils/ParallelListProcessor.cs index 9d544c6..9b67182 100644 --- a/src/AnalyticsEngine/Common/DataUtils/ParallelListProcessor.cs +++ b/src/AnalyticsEngine/Common/DataUtils/ParallelListProcessor.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Common.DataUtils +namespace DataUtils { /// /// Process a big list of objects in parallel, via configurable chunk sizes. diff --git a/src/AnalyticsEngine/Common/DataUtils/ResourceReader.cs b/src/AnalyticsEngine/Common/DataUtils/ProjectResourceReader.cs similarity index 86% rename from src/AnalyticsEngine/Common/DataUtils/ResourceReader.cs rename to src/AnalyticsEngine/Common/DataUtils/ProjectResourceReader.cs index fd335c0..e90e625 100644 --- a/src/AnalyticsEngine/Common/DataUtils/ResourceReader.cs +++ b/src/AnalyticsEngine/Common/DataUtils/ProjectResourceReader.cs @@ -4,18 +4,18 @@ using System.Linq; using System.Reflection; -namespace Common.DataUtils +namespace DataUtils { - public class ResourceReader + public class ProjectResourceReader { private readonly Assembly _assembly; - public ResourceReader(Assembly assembly) + public ProjectResourceReader(Assembly assembly) { _assembly = assembly; } - public string ReadResourceStringFromExecutingAssembly(string resourcePath) + public string ReadResourceString(string resourcePath) { using (var stream = GetAssemblyManifest(resourcePath)) diff --git a/src/AnalyticsEngine/Common/DataUtils/SecureLocalPreferences.cs b/src/AnalyticsEngine/Common/DataUtils/SecureLocalPreferences.cs index 6711fe5..d2fb565 100644 --- a/src/AnalyticsEngine/Common/DataUtils/SecureLocalPreferences.cs +++ b/src/AnalyticsEngine/Common/DataUtils/SecureLocalPreferences.cs @@ -2,7 +2,7 @@ using System; using System.IO; -namespace Common.DataUtils +namespace DataUtils { /// /// A class to load/save preferences securely on the local machine diff --git a/src/AnalyticsEngine/Common/DataUtils/Sql/Classes.cs b/src/AnalyticsEngine/Common/DataUtils/Sql/Classes.cs index 0e1e3e7..8a3e756 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Sql/Classes.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Sql/Classes.cs @@ -1,7 +1,7 @@ using System; using System.Data; -namespace Common.DataUtils.Sql +namespace DataUtils.Sql { public class ColumnSqlInfo { diff --git a/src/AnalyticsEngine/Common/DataUtils/Sql/Inserts/InsertBatch.cs b/src/AnalyticsEngine/Common/DataUtils/Sql/Inserts/InsertBatch.cs index a008cb2..e609e7f 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Sql/Inserts/InsertBatch.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Sql/Inserts/InsertBatch.cs @@ -6,7 +6,7 @@ using System.Reflection; using System.Threading.Tasks; -namespace Common.DataUtils.Sql.Inserts +namespace DataUtils.Sql.Inserts { /// /// For inserting & merging a lot of records quickly. Used when EF is too slow. diff --git a/src/AnalyticsEngine/Common/DataUtils/Sql/Inserts/InsertBatchTypeFieldCache.cs b/src/AnalyticsEngine/Common/DataUtils/Sql/Inserts/InsertBatchTypeFieldCache.cs index 29e0a78..291d441 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Sql/Inserts/InsertBatchTypeFieldCache.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Sql/Inserts/InsertBatchTypeFieldCache.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Reflection; -namespace Common.DataUtils.Sql.Inserts +namespace DataUtils.Sql.Inserts { /// /// Build SQL defintion from a InsertBatchType class diff --git a/src/AnalyticsEngine/Common/DataUtils/Sql/SqlHelper.cs b/src/AnalyticsEngine/Common/DataUtils/Sql/SqlHelper.cs index baf883e..6d28780 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Sql/SqlHelper.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Sql/SqlHelper.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Data; -namespace Common.DataUtils.Sql +namespace DataUtils.Sql { /// /// Turn .Net types into generic SQL types diff --git a/src/AnalyticsEngine/Common/DataUtils/Sql/SqlParamsExtension.cs b/src/AnalyticsEngine/Common/DataUtils/Sql/SqlParamsExtension.cs index 2739070..7057b3e 100644 --- a/src/AnalyticsEngine/Common/DataUtils/Sql/SqlParamsExtension.cs +++ b/src/AnalyticsEngine/Common/DataUtils/Sql/SqlParamsExtension.cs @@ -1,7 +1,7 @@ using System; using System.Data; -namespace Common.DataUtils.Sql +namespace DataUtils.Sql { public static class SqlParamsExtension { diff --git a/src/AnalyticsEngine/Common/DataUtils/StringCipher.cs b/src/AnalyticsEngine/Common/DataUtils/StringCipher.cs index 2c7a480..190d5f8 100644 --- a/src/AnalyticsEngine/Common/DataUtils/StringCipher.cs +++ b/src/AnalyticsEngine/Common/DataUtils/StringCipher.cs @@ -4,7 +4,7 @@ using System.Security.Cryptography; using System.Text; -namespace Common.DataUtils +namespace DataUtils { public static class StringCipher diff --git a/src/AnalyticsEngine/Common/DataUtils/StringUtils.cs b/src/AnalyticsEngine/Common/DataUtils/StringUtils.cs index 50dbb8d..0a61070 100644 --- a/src/AnalyticsEngine/Common/DataUtils/StringUtils.cs +++ b/src/AnalyticsEngine/Common/DataUtils/StringUtils.cs @@ -8,7 +8,7 @@ using System.Text.RegularExpressions; using System.Web; -namespace Common.DataUtils +namespace DataUtils { public static class StringUtils { @@ -34,7 +34,7 @@ public static bool IsValidAbsoluteUrl(string url) public static string GetOnlineMeetingId(string copilotDocContextId, string userGuid) { - var meetingIdFragment = StringUtils.GetMeetingIdFragmentFromMeetingThreadUrl(copilotDocContextId); + var meetingIdFragment = GetMeetingIdFragmentFromMeetingThreadUrl(copilotDocContextId); if (meetingIdFragment == null) { throw new Exception($"Could not parse meeting id from url {copilotDocContextId}"); @@ -58,7 +58,7 @@ public static string GetDriveItemId(string copilotDocContextId) } public static string GetSiteUrl(string copilotDocContextId) { - if (!StringUtils.IsValidAbsoluteUrl(copilotDocContextId)) + if (!IsValidAbsoluteUrl(copilotDocContextId)) { return null; } diff --git a/src/AnalyticsEngine/Common/DataUtils/UrlFilterProcessor.cs b/src/AnalyticsEngine/Common/DataUtils/UrlFilterProcessor.cs index 9fc5e29..04830ee 100644 --- a/src/AnalyticsEngine/Common/DataUtils/UrlFilterProcessor.cs +++ b/src/AnalyticsEngine/Common/DataUtils/UrlFilterProcessor.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace Common.DataUtils +namespace DataUtils { public class FilterUrlConfig { diff --git a/src/AnalyticsEngine/Common/Entities/Config/SiteFilterLoader.cs b/src/AnalyticsEngine/Common/Entities/Config/SiteFilterLoader.cs index 9d24232..c3fab08 100644 --- a/src/AnalyticsEngine/Common/Entities/Config/SiteFilterLoader.cs +++ b/src/AnalyticsEngine/Common/Entities/Config/SiteFilterLoader.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System.Collections.Generic; using System.Data.Entity; using System.Linq; diff --git a/src/AnalyticsEngine/Common/Entities/EFInsertBatch.cs b/src/AnalyticsEngine/Common/Entities/EFInsertBatch.cs index 1078485..33a29b5 100644 --- a/src/AnalyticsEngine/Common/Entities/EFInsertBatch.cs +++ b/src/AnalyticsEngine/Common/Entities/EFInsertBatch.cs @@ -1,4 +1,4 @@ -using Common.DataUtils.Sql.Inserts; +using DataUtils.Sql.Inserts; using Microsoft.Extensions.Logging; using System.Data.Entity; diff --git a/src/AnalyticsEngine/Common/Entities/Entities/Site.cs b/src/AnalyticsEngine/Common/Entities/Entities/Site.cs index 4d9fda8..39459f0 100644 --- a/src/AnalyticsEngine/Common/Entities/Entities/Site.cs +++ b/src/AnalyticsEngine/Common/Entities/Entities/Site.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; diff --git a/src/AnalyticsEngine/Common/Entities/Entities/Url.cs b/src/AnalyticsEngine/Common/Entities/Entities/Url.cs index bbc79c0..199c847 100644 --- a/src/AnalyticsEngine/Common/Entities/Entities/Url.cs +++ b/src/AnalyticsEngine/Common/Entities/Entities/Url.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; using Common.Entities.Entities; +using DataUtils; using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations.Schema; diff --git a/src/AnalyticsEngine/Common/Entities/Entities/Web.cs b/src/AnalyticsEngine/Common/Entities/Entities/Web.cs index dcb72f4..f33cc84 100644 --- a/src/AnalyticsEngine/Common/Entities/Entities/Web.cs +++ b/src/AnalyticsEngine/Common/Entities/Entities/Web.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; diff --git a/src/AnalyticsEngine/Common/Entities/Installer/BaseConfig.cs b/src/AnalyticsEngine/Common/Entities/Installer/BaseConfig.cs index 960b981..9f696f9 100644 --- a/src/AnalyticsEngine/Common/Entities/Installer/BaseConfig.cs +++ b/src/AnalyticsEngine/Common/Entities/Installer/BaseConfig.cs @@ -1,5 +1,4 @@ -using Common.DataUtils; -using System; +using DataUtils; using System.Collections.Generic; namespace Common.Entities.Installer diff --git a/src/AnalyticsEngine/Common/Entities/LookupCaches/Discrete/DBLookupCache.cs b/src/AnalyticsEngine/Common/Entities/LookupCaches/Discrete/DBLookupCache.cs index b65caeb..e6479fd 100644 --- a/src/AnalyticsEngine/Common/Entities/LookupCaches/Discrete/DBLookupCache.cs +++ b/src/AnalyticsEngine/Common/Entities/LookupCaches/Discrete/DBLookupCache.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System; using System.Data.Entity; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/Common/Entities/LookupCaches/SPWebResolver.cs b/src/AnalyticsEngine/Common/Entities/LookupCaches/SPWebResolver.cs index 3041621..a8a1a70 100644 --- a/src/AnalyticsEngine/Common/Entities/LookupCaches/SPWebResolver.cs +++ b/src/AnalyticsEngine/Common/Entities/LookupCaches/SPWebResolver.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System.Collections.Generic; using System.Data.Entity; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/Common/Entities/LookupCaches/TeamsAndCallsDBLookupManager.cs b/src/AnalyticsEngine/Common/Entities/LookupCaches/TeamsAndCallsDBLookupManager.cs index 3f5f626..9ebb2e1 100644 --- a/src/AnalyticsEngine/Common/Entities/LookupCaches/TeamsAndCallsDBLookupManager.cs +++ b/src/AnalyticsEngine/Common/Entities/LookupCaches/TeamsAndCallsDBLookupManager.cs @@ -1,8 +1,8 @@ -using Common.DataUtils; -using Common.Entities.Entities; +using Common.Entities.Entities; using Common.Entities.Entities.Teams; using Common.Entities.LookupCaches; using Common.Entities.Teams; +using DataUtils; using System; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/Common/UsageReporting/AnonUsageStatsModel.cs b/src/AnalyticsEngine/Common/UsageReporting/AnonUsageStatsModel.cs index b514ee1..af9cfd6 100644 --- a/src/AnalyticsEngine/Common/UsageReporting/AnonUsageStatsModel.cs +++ b/src/AnalyticsEngine/Common/UsageReporting/AnonUsageStatsModel.cs @@ -1,9 +1,9 @@ -using Common.DataUtils; +using DataUtils; using Newtonsoft.Json; using System; using System.Collections.Generic; -namespace Common.UsageReporting +namespace UsageReporting { /// /// Something that has to go in Cosmos DB @@ -42,7 +42,7 @@ public class TableStat { public string TableName { get; set; } public decimal TotalSpaceMB { get; set; } - public Int64 Rows { get; set; } + public long Rows { get; set; } public override string ToString() { @@ -59,27 +59,27 @@ public AnonUsageStatsModel UpdateWith(AnonUsageStatsModel updateFromClientWithNe { if (updateFromClientWithNewId.TableStats != null) { - this.TableStats = new List(updateFromClientWithNewId.TableStats); + TableStats = new List(updateFromClientWithNewId.TableStats); } if (!string.IsNullOrEmpty(updateFromClientWithNewId.ConfiguredImportsEnabledDescription)) { - this.ConfiguredImportsEnabledDescription = updateFromClientWithNewId.ConfiguredImportsEnabledDescription; + ConfiguredImportsEnabledDescription = updateFromClientWithNewId.ConfiguredImportsEnabledDescription; } if (!string.IsNullOrEmpty(updateFromClientWithNewId.ConfiguredSolutionsEnabledDescription)) { - this.ConfiguredSolutionsEnabledDescription = updateFromClientWithNewId.ConfiguredSolutionsEnabledDescription; + ConfiguredSolutionsEnabledDescription = updateFromClientWithNewId.ConfiguredSolutionsEnabledDescription; } if (!string.IsNullOrEmpty(updateFromClientWithNewId.BuildVersionLabel)) { - this.BuildVersionLabel = updateFromClientWithNewId.BuildVersionLabel; + BuildVersionLabel = updateFromClientWithNewId.BuildVersionLabel; } if (updateFromClientWithNewId.DataPointsFromAITotal.HasValue) { - this.DataPointsFromAITotal = updateFromClientWithNewId.DataPointsFromAITotal; + DataPointsFromAITotal = updateFromClientWithNewId.DataPointsFromAITotal; } - this.Generated = updateFromClientWithNewId.Generated; + Generated = updateFromClientWithNewId.Generated; } return this; @@ -96,7 +96,7 @@ public override string ToString() public bool IsValidSecretForThisObject(string cipher, string sharedSecretValue) { CheckEncryptionParamsValid(); - return StringUtils.IsHashedMatch(sharedSecretValue + this.Generated.Value.Ticks, cipher); + return StringUtils.IsHashedMatch(sharedSecretValue + Generated.Value.Ticks, cipher); } /// @@ -105,12 +105,12 @@ public bool IsValidSecretForThisObject(string cipher, string sharedSecretValue) public string GenerateSecretFromObjectProps(string sharedSecretValue) { CheckEncryptionParamsValid(); - return StringUtils.GetHashedStringWithSalt(sharedSecretValue + this.Generated.Value.Ticks); + return StringUtils.GetHashedStringWithSalt(sharedSecretValue + Generated.Value.Ticks); } private void CheckEncryptionParamsValid() { - if (!this.Generated.HasValue) + if (!Generated.HasValue) { throw new InvalidOperationException("Invalid "); } @@ -129,7 +129,7 @@ public HistoricalUpdate() public HistoricalUpdate(AnonUsageStatsModel update) : this() { Update = update; - this.id = Update != null && Update.Generated != null && Update.Generated.HasValue ? $"{Update.AnonClientId}-{Update.Generated.Value.Ticks}" : null; + id = Update != null && Update.Generated != null && Update.Generated.HasValue ? $"{Update.AnonClientId}-{Update.Generated.Value.Ticks}" : null; } @@ -145,8 +145,8 @@ public TelemetryPayload() public TelemetryPayload(AnonUsageStatsModel stats, string sharedSecretValue) : this() { - this.StatsModel = stats; - this.Secret = stats.GenerateSecretFromObjectProps(sharedSecretValue); + StatsModel = stats; + Secret = stats.GenerateSecretFromObjectProps(sharedSecretValue); } public AnonUsageStatsModel StatsModel { get; set; } diff --git a/src/AnalyticsEngine/Common/UsageReporting/CosmosTelemetrySaveAdaptor.cs b/src/AnalyticsEngine/Common/UsageReporting/CosmosTelemetrySaveAdaptor.cs index afc161f..4ca7a32 100644 --- a/src/AnalyticsEngine/Common/UsageReporting/CosmosTelemetrySaveAdaptor.cs +++ b/src/AnalyticsEngine/Common/UsageReporting/CosmosTelemetrySaveAdaptor.cs @@ -1,7 +1,7 @@ using Microsoft.Azure.Cosmos; using System.Threading.Tasks; -namespace Common.UsageReporting +namespace UsageReporting { public interface IStatsServiceCosmosConfig { diff --git a/src/AnalyticsEngine/Common/UsageReporting/TelemetrySaveAdaptor.cs b/src/AnalyticsEngine/Common/UsageReporting/TelemetrySaveAdaptor.cs index 18fe807..7f84e20 100644 --- a/src/AnalyticsEngine/Common/UsageReporting/TelemetrySaveAdaptor.cs +++ b/src/AnalyticsEngine/Common/UsageReporting/TelemetrySaveAdaptor.cs @@ -2,7 +2,7 @@ using System; using System.Threading.Tasks; -namespace Common.UsageReporting +namespace UsageReporting { /// /// Something that save telemetry data somewhere diff --git a/src/AnalyticsEngine/Tests.UnitTests/ActivityImporterTests.cs b/src/AnalyticsEngine/Tests.UnitTests/ActivityImporterTests.cs index 5131e83..eccb6e8 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/ActivityImporterTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/ActivityImporterTests.cs @@ -1,9 +1,9 @@ -using Common.DataUtils; -using Common.DataUtils.Http; -using Common.Entities; +using Common.Entities; using Common.Entities.Config; using Common.Entities.Entities; using Common.Entities.Entities.AuditLog; +using DataUtils; +using DataUtils.Http; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json; using System; @@ -19,7 +19,6 @@ using WebJob.Office365ActivityImporter.Engine; using WebJob.Office365ActivityImporter.Engine.ActivityAPI; using WebJob.Office365ActivityImporter.Engine.ActivityAPI.Loaders; -using WebJob.Office365ActivityImporter.Engine.Engine.Entities; using WebJob.Office365ActivityImporter.Engine.Entities; using WebJob.Office365ActivityImporter.Engine.Entities.Serialisation; diff --git a/src/AnalyticsEngine/Tests.UnitTests/AppInsightsImportTests.cs b/src/AnalyticsEngine/Tests.UnitTests/AppInsightsImportTests.cs index de0012b..ec910e5 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/AppInsightsImportTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/AppInsightsImportTests.cs @@ -1,9 +1,9 @@ using Azure; using Azure.AI.TextAnalytics; -using Common.DataUtils; using Common.Entities; using Common.Entities.Config; using Common.Entities.Entities; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json; diff --git a/src/AnalyticsEngine/Tests.UnitTests/CallRecordTests.cs b/src/AnalyticsEngine/Tests.UnitTests/CallRecordTests.cs index 029b7df..acaf4fd 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/CallRecordTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/CallRecordTests.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; +using DataUtils; using Microsoft.Graph; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json; diff --git a/src/AnalyticsEngine/Tests.UnitTests/CopilotTests.cs b/src/AnalyticsEngine/Tests.UnitTests/CopilotTests.cs index 38ca0e5..10fc3da 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/CopilotTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/CopilotTests.cs @@ -1,6 +1,6 @@ using ActivityImporter.Engine.ActivityAPI.Copilot; -using Common.DataUtils; using Common.Entities; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; @@ -108,7 +108,7 @@ public async Task CopilotEventManagerSaveTest() var outlook = new CopilotEventData { AppHost = "Outlook", - AccessedResources = new List + AccessedResources = new List { new AccessedResource{ Type = "http://schema.skype.com/HyperLink" } }, diff --git a/src/AnalyticsEngine/Tests.UnitTests/DataUtilsTests.cs b/src/AnalyticsEngine/Tests.UnitTests/DataUtilsTests.cs index b864a98..cc47a61 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/DataUtilsTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/DataUtilsTests.cs @@ -1,10 +1,10 @@ using App.ControlPanel.Engine; using App.ControlPanel.Engine.Models; -using Common.DataUtils; -using Common.DataUtils.Sql; using Common.Entities; using Common.Entities.Config; using Common.Entities.LookupCaches; +using DataUtils; +using DataUtils.Sql; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Concurrent; diff --git a/src/AnalyticsEngine/Tests.UnitTests/FakeControllers/FakeCallsController.cs b/src/AnalyticsEngine/Tests.UnitTests/FakeControllers/FakeCallsController.cs index bcd088d..79c32a4 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/FakeControllers/FakeCallsController.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/FakeControllers/FakeCallsController.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils; using Microsoft.Graph; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/FakeActivityImporter.cs b/src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/FakeActivityImporter.cs index 0605941..34c4546 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/FakeActivityImporter.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/FakeActivityImporter.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils; using WebJob.Office365ActivityImporter.Engine.ActivityAPI; namespace Tests.UnitTests.FakeLoaderClasses diff --git a/src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/FakeUserAppLoader.cs b/src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/FakeUserAppLoader.cs index 43f8a32..c9d9c8d 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/FakeUserAppLoader.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/FakeUserAppLoader.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System; using System.Collections.Generic; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/Tests.UnitTests/GraphImportTests.cs b/src/AnalyticsEngine/Tests.UnitTests/GraphImportTests.cs index fea974c..08c9d3d 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/GraphImportTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/GraphImportTests.cs @@ -1,9 +1,9 @@ using Azure.Messaging.ServiceBus; -using Common.DataUtils; using Common.Entities; using Common.Entities.Config; using Common.Entities.Entities.Teams; using Common.Entities.Models; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using Microsoft.SharePoint.Client; diff --git a/src/AnalyticsEngine/Tests.UnitTests/GraphUsageReportImportTests.cs b/src/AnalyticsEngine/Tests.UnitTests/GraphUsageReportImportTests.cs index 53e6628..a8b170d 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/GraphUsageReportImportTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/GraphUsageReportImportTests.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Linq; diff --git a/src/AnalyticsEngine/Tests.UnitTests/InstallTests/InstallEngineTests.cs b/src/AnalyticsEngine/Tests.UnitTests/InstallTests/InstallEngineTests.cs index 43c3dc8..2e2d96e 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/InstallTests/InstallEngineTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/InstallTests/InstallEngineTests.cs @@ -8,7 +8,6 @@ using App.ControlPanel.Engine.SharePointModelBuilder.ValueLookups; using Azure.Core; using Azure.Identity; -using Azure.ResourceManager.Resources.Models; using CloudInstallEngine; using CloudInstallEngine.Models; using Common.Entities.Config; @@ -146,7 +145,7 @@ public void TaskConfigTests() { { "testKey", "testVal" } }; - var cObj2 = c.ToArmParamsObject(new { tagsArray = new { value = tagsDict }}); + var cObj2 = c.ToArmParamsObject(new { tagsArray = new { value = tagsDict } }); Assert.IsNotNull(cObj2); diff --git a/src/AnalyticsEngine/Tests.UnitTests/Program.cs b/src/AnalyticsEngine/Tests.UnitTests/Program.cs index e8d4e1e..c8fca64 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/Program.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/Program.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils; using Microsoft.ApplicationInsights; using System; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/Tests.UnitTests/TeamsTests.cs b/src/AnalyticsEngine/Tests.UnitTests/TeamsTests.cs index 1122cfd..01c85b8 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/TeamsTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/TeamsTests.cs @@ -1,8 +1,8 @@ using Azure; using Azure.AI.TextAnalytics; -using Common.DataUtils; using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.Graph; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; diff --git a/src/AnalyticsEngine/Tests.UnitTests/UsageStatsTests.cs b/src/AnalyticsEngine/Tests.UnitTests/UsageStatsTests.cs index 2409a08..2f9eef5 100644 --- a/src/AnalyticsEngine/Tests.UnitTests/UsageStatsTests.cs +++ b/src/AnalyticsEngine/Tests.UnitTests/UsageStatsTests.cs @@ -1,7 +1,6 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.Installer; -using Common.UsageReporting; +using DataUtils; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -11,6 +10,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using UsageReporting; using WebJob.Office365ActivityImporter.Engine.StatsUploader; namespace Tests.UnitTests diff --git a/src/AnalyticsEngine/Web/Controllers/CallRecordWebhookController.cs b/src/AnalyticsEngine/Web/Controllers/CallRecordWebhookController.cs index 8f815db..a70d5b7 100644 --- a/src/AnalyticsEngine/Web/Controllers/CallRecordWebhookController.cs +++ b/src/AnalyticsEngine/Web/Controllers/CallRecordWebhookController.cs @@ -1,7 +1,7 @@ using Azure.Messaging.ServiceBus; -using Common.DataUtils; using Common.Entities.Config; using Common.Entities.Models; +using DataUtils; using Microsoft.Graph; using System.Collections.Generic; using System.Net; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/CustomEventsResultCollection.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/CustomEventsResultCollection.cs index 4a73336..d036a8f 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/CustomEventsResultCollection.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/CustomEventsResultCollection.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/EventCustomPropClasses.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/EventCustomPropClasses.cs index 2fedf77..03fbaa4 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/EventCustomPropClasses.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/EventCustomPropClasses.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/PageUpdateEventAppInsightsQueryResult.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/PageUpdateEventAppInsightsQueryResult.cs index 78dde2b..97e6bc7 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/PageUpdateEventAppInsightsQueryResult.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/APIResponseParsers/CustomEvents/PageUpdateEventAppInsightsQueryResult.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using Newtonsoft.Json; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/AppInsightsImporter.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/AppInsightsImporter.cs index b849dae..fe0562f 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/AppInsightsImporter.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/AppInsightsImporter.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Data.Entity; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/PageUpdates/PageCommentsExtensions.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/PageUpdates/PageCommentsExtensions.cs index c74c838..17017c5 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/PageUpdates/PageCommentsExtensions.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/PageUpdates/PageCommentsExtensions.cs @@ -1,5 +1,5 @@ -using Common.DataUtils.Sql; -using Common.Entities; +using Common.Entities; +using DataUtils.Sql; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/PageUpdates/PageUpdateManager.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/PageUpdates/PageUpdateManager.cs index 106482f..7daa39e 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/PageUpdates/PageUpdateManager.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/PageUpdates/PageUpdateManager.cs @@ -1,10 +1,10 @@ using Azure; using Azure.AI.TextAnalytics; -using Common.DataUtils; using Common.Entities; using Common.Entities.Config; using Common.Entities.Entities; using Common.Entities.LookupCaches; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/ClickTempEntity.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/ClickTempEntity.cs index 7c58c4e..fa60801 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/ClickTempEntity.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/ClickTempEntity.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.DataUtils.Sql; +using DataUtils; +using DataUtils.Sql; using System; using WebJob.AppInsightsImporter.Engine.ApiImporter; using WebJob.AppInsightsImporter.Engine.APIResponseParsers.CustomEvents; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/HitTempEntity.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/HitTempEntity.cs index d4bf85c..90cdfd8 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/HitTempEntity.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/HitTempEntity.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.DataUtils.Sql; +using DataUtils; +using DataUtils.Sql; using System; using WebJob.AppInsightsImporter.Engine.ApiImporter; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/HitUpdate.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/HitUpdate.cs index 3df835a..711b42c 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/HitUpdate.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/Models/HitUpdate.cs @@ -1,4 +1,4 @@ -using Common.DataUtils.Sql; +using DataUtils.Sql; using System; using WebJob.AppInsightsImporter.Engine.APIResponseParsers.CustomEvents; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/PageUpdatesSaveExtension.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/PageUpdatesSaveExtension.cs index b7fa4b1..562e305 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/PageUpdatesSaveExtension.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/PageUpdatesSaveExtension.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.Entities; +using DataUtils; using Microsoft.Extensions.Logging; using System.Data.Entity; using System.Linq; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/PageViewsSaveExtension.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/PageViewsSaveExtension.cs index ea00769..0a3f740 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/PageViewsSaveExtension.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter.Engine/Sql/PageViewsSaveExtension.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.AppInsightsImporter/Program.cs b/src/AnalyticsEngine/WebJob.AppInsightsImporter/Program.cs index 273a58e..971e524 100644 --- a/src/AnalyticsEngine/WebJob.AppInsightsImporter/Program.cs +++ b/src/AnalyticsEngine/WebJob.AppInsightsImporter/Program.cs @@ -4,9 +4,9 @@ // IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A // PARTICULAR PURPOSE. -using Common.DataUtils; using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Configuration; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Abstract.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Abstract.cs index 3dc8806..2845b0b 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Abstract.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Abstract.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.DataUtils.Http; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils; +using DataUtils.Http; namespace WebJob.Office365ActivityImporter.Engine { diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityImporter.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityImporter.cs index de56291..1d4cfbd 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityImporter.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityImporter.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils; using System; using System.Collections.Generic; using System.Linq; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityReportSqlPersistenceManager.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityReportSqlPersistenceManager.cs index 6dbb058..2a7f03f 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityReportSqlPersistenceManager.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityReportSqlPersistenceManager.cs @@ -1,6 +1,6 @@ -using Common.DataUtils.Sql; -using Common.Entities; +using Common.Entities; using Common.Entities.Config; +using DataUtils.Sql; using Microsoft.Extensions.Logging; using System; using System.Collections.Concurrent; @@ -231,6 +231,11 @@ public AuditLogTempEntity(AbstractAuditLogContent abtractLog, string userNameOrH this.UrlBase = spLog.SiteUrl; this.EventData = spLog.EventData; } + + if (abtractLog is CopilotAuditLogContent) { + var copilotLog = (CopilotAuditLogContent)abtractLog; + this.EventData = copilotLog.EventRaw; + } } [Column("log_id")] diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivitySubscriptionManager.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivitySubscriptionManager.cs index bbfbbc8..256ba94 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivitySubscriptionManager.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivitySubscriptionManager.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.DataUtils.Http; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils; +using DataUtils.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/AuditFilterConfig.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/AuditFilterConfig.cs index f1bc3b2..54e54aa 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/AuditFilterConfig.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/AuditFilterConfig.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.Extensions.Logging; using System.Collections.Generic; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/CopilotAuditEventManager.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/CopilotAuditEventManager.cs index 8810321..98c7052 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/CopilotAuditEventManager.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/CopilotAuditEventManager.cs @@ -1,9 +1,10 @@ -using Common.DataUtils; -using Common.DataUtils.Sql.Inserts; -using Common.Entities; +using Common.Entities; using Common.Entities.Entities.AuditLog; +using DataUtils; +using DataUtils.Sql.Inserts; using Microsoft.Extensions.Logging; using System; +using System.Linq; using System.Threading.Tasks; using WebJob.Office365ActivityImporter.Engine; using WebJob.Office365ActivityImporter.Engine.ActivityAPI.Copilot; @@ -34,7 +35,6 @@ public CopilotAuditEventManager(string connectionString, ICopilotMetadataLoader _db = new AnalyticsEntitiesContext(); } - public async Task SaveSingleCopilotEventToSql(CopilotEventData eventData, Office365Event baseOfficeEvent) { _logger.LogInformation($"Saving copilot event metadata to SQL for event {baseOfficeEvent.Id}"); @@ -44,7 +44,8 @@ public async Task SaveSingleCopilotEventToSql(CopilotEventData eventData, Office if (eventData.Contexts != null && eventData.Contexts.Count > 0) { - // Process events with context (Teams meeting, file etc) + // Process events with context (Teams meeting, file etc). + // Normally only one context per event, but we'll loop through them all just in case. foreach (var context in eventData.Contexts) { // There are some known context types for Teams etc. Everything else is assumed to be a file type. @@ -72,6 +73,7 @@ public async Task SaveSingleCopilotEventToSql(CopilotEventData eventData, Office }); meetingsCount++; + break; // Only one meeting per event } else if (context.Type == ActivityImportConstants.COPILOT_CONTEXT_TYPE_TEAMS_CHAT) { @@ -101,6 +103,8 @@ public async Task SaveSingleCopilotEventToSql(CopilotEventData eventData, Office UrlBase = spFileInfo.SiteUrl }); filesCount++; + break; // Normally only one file per event. + // There can be more documents in the context if one references another, but we only care about the doc the user is in. } else { @@ -130,16 +134,16 @@ public async Task SaveSingleCopilotEventToSql(CopilotEventData eventData, Office public async Task CommitAllChanges() { - var rr = new ResourceReader(System.Reflection.Assembly.GetExecutingAssembly()); - var docsMergeSql = rr.ReadResourceStringFromExecutingAssembly("WebJob.Office365ActivityImporter.Engine.ActivityAPI.Copilot.SQL.insert_sp_copilot_events_from_staging_table.sql") + var rr = new ProjectResourceReader(System.Reflection.Assembly.GetExecutingAssembly()); + var docsMergeSql = rr.ReadResourceString("WebJob.Office365ActivityImporter.Engine.ActivityAPI.Copilot.SQL.insert_sp_copilot_events_from_staging_table.sql") .Replace(ActivityImportConstants.STAGING_TABLE_VARNAME, ActivityImportConstants.STAGING_TABLE_COPILOT_SP); - var teamsMergeSql = rr.ReadResourceStringFromExecutingAssembly("WebJob.Office365ActivityImporter.Engine.ActivityAPI.Copilot.SQL.insert_teams_copilot_events_from_staging_table.sql") + var teamsMergeSql = rr.ReadResourceString("WebJob.Office365ActivityImporter.Engine.ActivityAPI.Copilot.SQL.insert_teams_copilot_events_from_staging_table.sql") .Replace(ActivityImportConstants.STAGING_TABLE_VARNAME, ActivityImportConstants.STAGING_TABLE_COPILOT_TEAMS); - var chatOnlyMergeSql = rr.ReadResourceStringFromExecutingAssembly("WebJob.Office365ActivityImporter.Engine.ActivityAPI.Copilot.SQL.insert_chat_only_copilot_events_from_staging_table.sql") + var chatOnlyMergeSql = rr.ReadResourceString("WebJob.Office365ActivityImporter.Engine.ActivityAPI.Copilot.SQL.insert_chat_only_copilot_events_from_staging_table.sql") .Replace(ActivityImportConstants.STAGING_TABLE_VARNAME, ActivityImportConstants.STAGING_TABLE_COPILOT_CHATONLY); diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/GraphCaches.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/GraphCaches.cs index 8e3b0ab..b4c36ad 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/GraphCaches.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/GraphCaches.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using Microsoft.Graph; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/GraphFileMetadataLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/GraphFileMetadataLoader.cs index e64301c..54e06c6 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/GraphFileMetadataLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/GraphFileMetadataLoader.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using System; @@ -75,11 +75,10 @@ public async Task GetSpoFileInfo(string copilotDocContextId } var driveItemId = StringUtils.GetDriveItemId(copilotDocContextId); + ListItem item = null; + var site = await _siteGraphCache.GetResourceOrNullIfNotExists(spSiteId); if (driveItemId != null) { - var site = await _siteGraphCache.GetResourceOrNullIfNotExists(spSiteId); - - ListItem item = null; try { item = await _graphServiceClient.Sites[spSiteId].Lists[spListId].Items[driveItemId] @@ -95,6 +94,30 @@ public async Task GetSpoFileInfo(string copilotDocContextId } else { + // We might have a direct URL as the copilot context ID, so we need to search for the item in the list. + // Example: https://contoso-my.sharepoint.com/personal/alex_contoso_onmicrosoft_com/Documents/MyDoc.docx + try + { + // Currently we can't filter by webUrl, so we have to get all items and filter client side + var listItems = await _graphServiceClient.Sites[spSiteId].Lists[spListId].Items + .Request().Select("id,webUrl").GetAsync(); + if (listItems != null) + { + foreach (var i in listItems) + { + if (i.WebUrl == copilotDocContextId) + { + return new SpoDocumentFileInfo(i, site); + } + } + } + } + catch (ServiceException ex) + { + _logger.LogWarning(ex, "Error getting items info for list {spListId} on site {siteUrl}", spListId, siteUrl); + return null; + } + _logger.LogWarning("No driveItemId found in copilotDocContextId {copilotDocContextId}", copilotDocContextId); return null; } diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/StagingClasses.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/StagingClasses.cs index c884f10..43558c4 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/StagingClasses.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Copilot/StagingClasses.cs @@ -1,4 +1,4 @@ -using Common.DataUtils.Sql; +using DataUtils.Sql; using System; namespace WebJob.Office365ActivityImporter.Engine.ActivityAPI.Copilot diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/ActivityReportLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/ActivityReportLoader.cs index 5b97acc..531f85d 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/ActivityReportLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/ActivityReportLoader.cs @@ -1,4 +1,4 @@ -using Common.DataUtils.Http; +using DataUtils.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -92,6 +92,10 @@ public async Task Load(ActivityReportInfo metadata) try { thisAuditLogReport = JsonConvert.DeserializeObject(logJson); + // We want to store the CopilotEventData but its current schema may change in the future. Keeping the full CopilotEventData object for now. + var asCopilotReport = (CopilotAuditLogContent)thisAuditLogReport; + dynamic obj = JsonConvert.DeserializeObject(logJson); + asCopilotReport.EventRaw = JsonConvert.SerializeObject(obj.CopilotEventData); } catch (JsonReaderException) { diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/ActivityWebImporter.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/ActivityWebImporter.cs index e732455..586d4ae 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/ActivityWebImporter.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/ActivityWebImporter.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.DataUtils.Http; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils; +using DataUtils.Http; namespace WebJob.Office365ActivityImporter.Engine.ActivityAPI.Loaders { diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/WebContentMetaDataLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/WebContentMetaDataLoader.cs index ff9d0d5..1feee85 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/WebContentMetaDataLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/Loaders/WebContentMetaDataLoader.cs @@ -1,5 +1,5 @@ -using Common.DataUtils.Http; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityImportCache.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityImportCache.cs index ef01a62..00ae60b 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityImportCache.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityImportCache.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; +using DataUtils; using System; using System.Collections.Generic; using System.Linq; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/ImportStat.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/ImportStat.cs index 961b169..fc82b16 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/ImportStat.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/ImportStat.cs @@ -1,5 +1,4 @@ -using Microsoft.Graph; -using System; +using System; using System.Collections.Generic; using WebJob.Office365ActivityImporter.Engine.Engine.Entities; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/CopilotAuditLogContent.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/CopilotAuditLogContent.cs index d1ad788..709b070 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/CopilotAuditLogContent.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/CopilotAuditLogContent.cs @@ -1,6 +1,4 @@ using Common.Entities; -using Newtonsoft.Json.Linq; -using StackExchange.Redis; using System.Collections.Generic; using System.Threading.Tasks; using WebJob.Office365ActivityImporter.Engine.ActivityAPI; @@ -11,6 +9,7 @@ namespace WebJob.Office365ActivityImporter.Engine.Entities.Serialisation public class CopilotAuditLogContent : AbstractAuditLogContent { public CopilotEventData CopilotEventData { get; set; } = null; + public string EventRaw { get; set; } = null; public override async Task ProcessExtendedProperties(SaveSession sessionContext, Office365Event relatedAuditEvent) { await sessionContext.CopilotEventResolver.SaveSingleCopilotEventToSql(CopilotEventData, relatedAuditEvent); diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/SharePointAuditLogContent.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/SharePointAuditLogContent.cs index 0d3f17d..dd58b4d 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/SharePointAuditLogContent.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/SharePointAuditLogContent.cs @@ -1,6 +1,5 @@ -using Common.DataUtils; -using Common.Entities; -using System; +using Common.Entities; +using DataUtils; using System.Threading.Tasks; using WebJob.Office365ActivityImporter.Engine.ActivityAPI; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Calls/CallQueueProcessor.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Calls/CallQueueProcessor.cs index 25d939b..cc13bcb 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Calls/CallQueueProcessor.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Calls/CallQueueProcessor.cs @@ -1,9 +1,9 @@ using Azure.Messaging.ServiceBus; -using Common.DataUtils; -using Common.DataUtils.Http; using Common.Entities; using Common.Entities.Config; using Common.Entities.Models; +using DataUtils; +using DataUtils.Http; using Microsoft.Extensions.Logging; using Microsoft.Graph; using Newtonsoft.Json; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/GraphImporter.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/GraphImporter.cs index df5ca56..603adea 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/GraphImporter.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/GraphImporter.cs @@ -1,7 +1,7 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.ActivityReports; using Common.Entities.Config; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using System; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/GraphLookupCache.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/GraphLookupCache.cs index 4ac3bad..bfba11c 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/GraphLookupCache.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/GraphLookupCache.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using Microsoft.Graph; using System; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/ManualGraphCallClient.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/ManualGraphCallClient.cs index 19b0ec7..9c0b6ae 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/ManualGraphCallClient.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/ManualGraphCallClient.cs @@ -1,4 +1,4 @@ -using Common.DataUtils.Http; +using DataUtils.Http; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/SPSiteIdToUrlCache.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/SPSiteIdToUrlCache.cs index 7442d73..c2816a5 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/SPSiteIdToUrlCache.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/SPSiteIdToUrlCache.cs @@ -1,10 +1,9 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using System; using System.Data.Entity; -using System.Data.Entity.Infrastructure; using System.Linq; using System.Threading.Tasks; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/ChannelMessagesLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/ChannelMessagesLoader.cs index 6847b1f..57b9bee 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/ChannelMessagesLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/ChannelMessagesLoader.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities.Redis; +using Common.Entities.Redis; using Common.Entities.Redis.Teams; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using System; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/CognitiveHelper.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/CognitiveHelper.cs index 821560e..f9b9763 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/CognitiveHelper.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/CognitiveHelper.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using Microsoft.Graph; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/Extensions/TeamsCognitiveExtensions.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/Extensions/TeamsCognitiveExtensions.cs index 0ced378..dd53ab9 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/Extensions/TeamsCognitiveExtensions.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/Extensions/TeamsCognitiveExtensions.cs @@ -1,7 +1,7 @@ using Azure.AI.TextAnalytics; -using Common.DataUtils; using Common.Entities.Config; using Common.Entities.Redis; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using Newtonsoft.Json; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamsFinder.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamsFinder.cs index 455b2d7..ba1c64a 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamsFinder.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamsFinder.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities.Config; +using Common.Entities.Config; +using DataUtils; using Microsoft.Graph; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamsImporter.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamsImporter.cs index e2ce8e5..feacf27 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamsImporter.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamsImporter.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using System; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/UsageReports/AbstractDailyActivityLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/UsageReports/AbstractDailyActivityLoader.cs index fe94363..5b94123 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/UsageReports/AbstractDailyActivityLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/UsageReports/AbstractDailyActivityLoader.cs @@ -1,7 +1,7 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.ActivityReports; using Common.Entities.LookupCaches; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/GraphUserLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/GraphUserLoader.cs index 501329d..9d8e2af 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/GraphUserLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/GraphUserLoader.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities.Config; +using Common.Entities.Config; using Common.Entities.Redis; +using DataUtils; using Microsoft.Extensions.Logging; using System.Collections.Generic; using System.Linq; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/AbstractUserAppLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/AbstractUserAppLoader.cs index 43ca5a8..9e544fe 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/AbstractUserAppLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/AbstractUserAppLoader.cs @@ -1,4 +1,4 @@ -using Common.DataUtils; +using DataUtils; using System; using System.Collections.Generic; using System.Linq; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/GraphAndSqlUserAppLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/GraphAndSqlUserAppLoader.cs index acce8e5..97d4e84 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/GraphAndSqlUserAppLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/GraphAndSqlUserAppLoader.cs @@ -1,5 +1,5 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using Newtonsoft.Json; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/Models.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/Models.cs index 6fb55c1..d597980 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/Models.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/Models.cs @@ -1,4 +1,4 @@ -using Common.DataUtils.Sql; +using DataUtils.Sql; using Microsoft.Graph; using System; using System.Collections.Generic; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/UserAppLogUpdater.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/UserAppLogUpdater.cs index 334f5f9..d983563 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/UserAppLogUpdater.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserApps/UserAppLogUpdater.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.Graph; using System.Threading.Tasks; using WebJob.Office365ActivityImporter.Engine.Graph.User.UserApps; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserMetadataUpdater.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserMetadataUpdater.cs index 676f4c0..850d5b8 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserMetadataUpdater.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserMetadataUpdater.cs @@ -1,7 +1,7 @@ using Azure.Core; -using Common.DataUtils; using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.Extensions.Logging; using Microsoft.Graph; using System; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/OAuthContext.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/OAuthContext.cs index ec6211f..bf47422 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/OAuthContext.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/OAuthContext.cs @@ -1,4 +1,4 @@ -using Common.DataUtils.Http; +using DataUtils.Http; using Microsoft.Extensions.Logging; namespace WebJob.Office365ActivityImporter.Engine diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/AnonUsageStatsModelLoader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/AnonUsageStatsModelLoader.cs index eb9f564..9658e66 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/AnonUsageStatsModelLoader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/AnonUsageStatsModelLoader.cs @@ -1,7 +1,7 @@ -using Common.DataUtils; -using Common.Entities.Installer; -using Common.UsageReporting; +using Common.Entities.Installer; +using DataUtils; using System; +using UsageReporting; namespace WebJob.Office365ActivityImporter.Engine.StatsUploader { diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/BaseClasses.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/BaseClasses.cs index 0d4ba59..cdef5a1 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/BaseClasses.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/BaseClasses.cs @@ -1,8 +1,8 @@ using Common.Entities.Installer; -using Common.UsageReporting; using Microsoft.Extensions.Logging; using System; using System.Threading.Tasks; +using UsageReporting; namespace WebJob.Office365ActivityImporter.Engine.StatsUploader { diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/SqlUsageStatsBuilder.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/SqlUsageStatsBuilder.cs index 839d8de..0d24503 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/SqlUsageStatsBuilder.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/SqlUsageStatsBuilder.cs @@ -1,6 +1,5 @@ using Common.Entities; using Common.Entities.Installer; -using Common.UsageReporting; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System; @@ -8,6 +7,7 @@ using System.Data.Entity; using System.Linq; using System.Threading.Tasks; +using UsageReporting; namespace WebJob.Office365ActivityImporter.Engine.StatsUploader { diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/WebApiStatsUploader.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/WebApiStatsUploader.cs index 284e6e2..2d4faae 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/WebApiStatsUploader.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/StatsUploader/WebApiStatsUploader.cs @@ -1,10 +1,10 @@ -using Common.UsageReporting; -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System; using System.Net.Http; using System.Text; using System.Threading.Tasks; +using UsageReporting; namespace WebJob.Office365ActivityImporter.Engine.StatsUploader { diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter/AutomationPS/ProfilingJobs/Aggregation_Status.ps1 b/src/AnalyticsEngine/WebJob.Office365ActivityImporter/AutomationPS/ProfilingJobs/Aggregation_Status.ps1 index ac0ffc6..f2e308a 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter/AutomationPS/ProfilingJobs/Aggregation_Status.ps1 +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter/AutomationPS/ProfilingJobs/Aggregation_Status.ps1 @@ -24,28 +24,38 @@ Connection Timeout=60" $DatabaseConnection = New-Object System.Data.SqlClient.SqlConnection($SqlConnectionString) $SqlCmd = @" -SELECT s.name, t.name, p.rows FROM sys.partitions AS p - INNER JOIN sys.tables AS t ON p.[object_id] = t.[object_id] - INNER JOIN sys.schemas AS s ON s.[schema_id] = t.[schema_id] -WHERE (s.name = N'profiling' OR t.name LIKE '%_user_activity_log') AND p.index_id IN (0,1) -ORDER BY s.name, t.name; +-- Query 1 +SELECT schemas.name, tables.name, partitions.rows +FROM sys.partitions AS partitions + INNER JOIN sys.tables AS tables ON partitions.[object_id] = tables.[object_id] + INNER JOIN sys.schemas AS schemas ON schemas.[schema_id] = tables.[schema_id] +WHERE ( + schemas.name = N'profiling' + OR tables.name LIKE '%_user_activity_log' + OR tables.name LIKE 'event_copilot_%' +) AND partitions.index_id IN (0,1) +ORDER BY schemas.name, tables.name; +-- Query 2 SELECT MIN(MetricDate) WeeklyActivitiesFrom, MAX(MetricDate) WeeklyActivitiesTo FROM [profiling].[ActivitiesWeekly]; +-- Query 3 SELECT MIN([date]) WeeklyActivitiesColumnsFrom, MAX([date]) WeeklyActivitiesColumnsTo FROM [profiling].[ActivitiesWeeklyColumns]; +-- Query 4 SELECT MIN([date]) WeeklyUsageFrom, MAX([date]) WeeklyUsageTo FROM [profiling].[UsageWeekly]; "@ try { $DatabaseConnection.Open() - $Cmd = New-Object system.Data.SqlClient.SqlCommand + $Cmd = New-Object System.Data.SqlClient.SqlCommand $Cmd.Connection = $DatabaseConnection $Cmd.CommandTimeout = 10800 # 3 hours $Cmd.CommandText = $SqlCmd $results = $cmd.ExecuteReader() + # Query 1 Write-Output "Profiling table sizes" while ($results.Read()) { $schema = $results.GetString(0) @@ -53,32 +63,39 @@ try { $rows = $results.GetInt64(2) Write-Output "$schema.$($table): $rows rows" } - - $results.NextResult() | Out-Null - while ($results.Read()) { - $from = $results.GetDateTime(0) - $to = $results.GetDateTime(1) - Write-Output "ActivitiesWeekly from $(fd $from) to $(fd $to)" + Write-Output "" + # Query 2 + if ($results.NextResult()) { + while ($results.Read()) { + $from = $results.GetDateTime(0) + $to = $results.GetDateTime(1) + Write-Output "ActivitiesWeekly from $(fd $from) to $(fd $to)" + } + Write-Output "" } - $results.NextResult() | Out-Null - while ($results.Read()) { - $from = $results.GetDateTime(0) - $to = $results.GetDateTime(1) - Write-Output "ActivitiesWeeklyColumns from $(fd $from) to $(fd $to)" + # Query 3 + if ($results.NextResult()) { + while ($results.Read()) { + $from = $results.GetDateTime(0) + $to = $results.GetDateTime(1) + Write-Output "ActivitiesWeeklyColumns from $(fd $from) to $(fd $to)" + } + Write-Output "" } - $results.NextResult() | Out-Null - while ($results.Read()) { - $from = $results.GetDateTime(0) - $to = $results.GetDateTime(1) - Write-Output "UsageWeekly from $(fd $from) to $(fd $to)" + # Query 4 + if ($results.NextResult()) { + while ($results.Read()) { + $from = $results.GetDateTime(0) + $to = $results.GetDateTime(1) + Write-Output "UsageWeekly from $(fd $from) to $(fd $to)" + } + Write-Output "" } $results.Close() -} -catch { +} catch { $LastError = $_.Exception -} -finally { +} finally { $DatabaseConnection.Close() $DatabaseConnection.Dispose() } diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter/Program.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter/Program.cs index 1de822b..8a170f8 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter/Program.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter/Program.cs @@ -5,10 +5,10 @@ // PARTICULAR PURPOSE. #region Usings -using Common.DataUtils; using Common.Entities; using Common.Entities.Config; using Common.Entities.Installer; +using DataUtils; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System; diff --git a/src/AnalyticsEngine/WebJob.Office365ActivityImporter/ProgramTasks.cs b/src/AnalyticsEngine/WebJob.Office365ActivityImporter/ProgramTasks.cs index a14c162..281cfcb 100644 --- a/src/AnalyticsEngine/WebJob.Office365ActivityImporter/ProgramTasks.cs +++ b/src/AnalyticsEngine/WebJob.Office365ActivityImporter/ProgramTasks.cs @@ -1,6 +1,6 @@ -using Common.DataUtils; -using Common.Entities; +using Common.Entities; using Common.Entities.Config; +using DataUtils; using Microsoft.Extensions.Logging; using System; using System.Threading.Tasks; diff --git a/src/SPO/AITracker/.gitignore b/src/SPO/AITracker/.gitignore new file mode 100644 index 0000000..aa894a7 --- /dev/null +++ b/src/SPO/AITracker/.gitignore @@ -0,0 +1,2 @@ + +/TypeScript/node_modules \ No newline at end of file