diff --git a/samples/manage/azure-sql-db-elastic-pools/Pool Telemetry PS User Guide.docx b/samples/manage/azure-sql-db-elastic-pools/Pool Telemetry PS User Guide.docx new file mode 100644 index 0000000000..b8e1b65b1e Binary files /dev/null and b/samples/manage/azure-sql-db-elastic-pools/Pool Telemetry PS User Guide.docx differ diff --git a/samples/manage/azure-sql-db-elastic-pools/PoolTelemetry.ps1 b/samples/manage/azure-sql-db-elastic-pools/PoolTelemetry.ps1 new file mode 100644 index 0000000000..cbf2b40b19 --- /dev/null +++ b/samples/manage/azure-sql-db-elastic-pools/PoolTelemetry.ps1 @@ -0,0 +1,219 @@ +# ---------------------------------------------------------------------------------- +# +# Copyright Microsoft Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------------- +# +# Sample script for loading SQL Database telemetry for pools and elastic databases on a single server +# to a user-supplied Azure SQL database. +# +# +# See additional comments in PoolTelemetryRunner.ps1, which includes script and instructions for running +# this script as a PowerShell job, enabling data gathering for large numbers of servers in the background. +# +# +function Load-PoolTelemetryForServer { + param ( + [Parameter(Mandatory=$true)][string]$SubscriptionId, # Azure subscription owning the server from which telemetry will be gathered - see https://portal.azure.com + [Parameter(Mandatory=$true)][string]$ResourceGroupName, # name of resource group containing the server from which telemetry will be gathered - see https://portal.azure.com + [Parameter(Mandatory=$true)][string]$ServerName, # name of server from which telemetry will be gathered - e.g. "myappserver" + [Parameter(Mandatory=$true)][string]$Location, # location of server, e.g. "Australia Southeast" + [Parameter(Mandatory=$true)][PSCredential]$ServerCred, + [Parameter(Mandatory=$true)][string]$OutputServerName, # server name of telemetry database, like "telemetryserver" + [Parameter(Mandatory=$true)][string]$OutputDatabaseName, # telemetry database name, like "telemetrydb" + [Parameter(Mandatory=$true)][PSCredential]$OutputServerCred, + [Parameter(Mandatory=$true)][int]$IntervalMinutes, # interval for collection of telemetry + [Parameter(Mandatory=$true)][int]$DurationMinutes, # total duration for collection of telemetry + [Parameter(Mandatory=$true)][bool]$IncludeDatabases # indicates if telemetry should be gathered for databases as well as pools + ) + + # Create output database metrics collection tables if it does not already exist. + + $OutputServerCred.Password.MakeReadOnly() + $sqlCred = new-object ("System.Data.SqlClient.SqlCredential") -ArgumentList $OutputServerCred.UserName, $OutputServerCred.Password + + $outputConnection = New-Object ("System.Data.SqlClient.SqlConnection") "Data Source=$OutputServerName.database.windows.net;Integrated Security=false;Initial Catalog=$OutputDatabaseName" + $outputConnection.Credential = $sqlCred + $outputConnection.Open() + + $poolResourceStatsTable = "pool_resource_stats" # <<< update if a different table name is required + $dbResourceStatsTable = "db_resource_stats" # <<< update if a different table name is required + + $sql =` + "-- Create table for holding collected pool resource stats + IF NOT EXISTS (SELECT * FROM sys.objects + WHERE object_id = OBJECT_ID(N'$($poolResourceStatsTable)') AND type in (N'U')) + + BEGIN + Create Table $($poolResourceStatsTable) (subscription_guid uniqueidentifier, resource_group_name varchar(128), server_name varchar(128), location varchar(128), elastic_pool_name varchar(128), end_time datetime, + elastic_pool_DTU_limit int, avg_cpu_percent decimal(5,2), avg_data_io_percent decimal(5,2), avg_log_io_percent decimal(5,2), max_worker_percent decimal(5,2), max_session_percent decimal(5,2) + , avg_DTU_percent decimal(5,2), avg_storage_percent decimal(5,2), elastic_pool_storage_limit_mb bigint); + Create Clustered Index ci_endtime ON $($poolResourceStatsTable) (end_time); + END + + -- Create table for holding collected database resource stats + IF NOT EXISTS (SELECT * FROM sys.objects + WHERE object_id = OBJECT_ID(N'$($dbResourceStatsTable)') AND type in (N'U')) + + BEGIN + Create Table $($dbResourceStatsTable) (subscription_guid uniqueidentifier, resource_group_name varchar(128),server_name varchar(128), location varchar(128), elastic_pool_name varchar(128), database_name varchar(128), end_time datetime, + database_DTU_limit int, avg_cpu_percent decimal(5,2), avg_data_io_percent decimal(5,2), avg_log_io_percent decimal(5,2), max_worker_percent decimal(5,2), max_session_percent decimal(5,2), avg_DTU_percent decimal(5,2), db_size float); + Create Clustered Index ci_endtime ON $($dbResourceStatsTable) (end_time); + END + + -- Create a function to get aggregated metrics for a given time interval + IF NOT EXISTS (SELECT * FROM sys.objects + WHERE name = N'get_aggregated_pool_metrics' AND type in ('IF')) + EXEC sp_executesql @Statement = N' + Create function get_aggregated_pool_metrics( + @start datetime + ,@end datetime) + RETURNS TABLE + AS + RETURN + ( + SELECT + location, server_name, elastic_pool_name + ,avg([avg_dtu_percent]) as avg_eDTU_percent + ,avg([avg_cpu_percent]) as avg_cpu_percent + ,avg([avg_data_io_percent]) as avg_data_io_percent + ,avg([avg_log_io_percent]) as avg_log_io_percent + ,avg([avg_storage_percent]) as avg_storage_percent + ,max([avg_dtu_percent]) as max_of_avg_eDTU_percent + ,max([avg_cpu_percent]) as max_of_avg_cpu_percent + ,max([avg_data_io_percent]) as max_of_data_io_percent + ,max([avg_log_io_percent]) as max_of_avg_log_io_percent + ,max([avg_storage_percent]) as max_of_avg_storage_percent + ,max([max_worker_percent]) as max_workers_percent + ,max([max_session_percent]) as max_session_percent + FROM [dbo].[pool_resource_stats] + WHERE end_time between @start and @end + group by location, server_name, elastic_pool_name + )' + " + + + $outputServerFullname = $OutputServerName + '.database.windows.net' # assumes server is in Azure SQL Database + + Invoke-Sqlcmd -ServerInstance $outputServerFullName -Database $OutputDatabaseName -Username $OutputServerCred.UserName -Password $OutputServerCred.GetNetworkCredential().Password -Query $sql -ConnectionTimeout 120 -QueryTimeout 120 + + $sourceServerFullName = $ServerName + '.database.windows.net' + + $interval = $IntervalMinutes + $now = [DateTime]::UtcNow + [DateTime]$startTime = $now.AddMinutes(-$interval) # sets the start time for the first telemetry collection + [DateTime]$endTime = $now # sets the end time + [DateTime]$finishTime = $now.AddMinutes($DurationMinutes) # sets the overall finish time for a telemetry collection session + + Write-Host "Starting to collect telemetry for" $ServerName + + $poolLagMinutes = 30 # This accommodates the lag in pool-level metrics being available in the DMV, does not affect per database telemetry + + while($startTime -lt $finishTime) + { + $poolStartTime = $startTime.AddMinutes(-$poolLagMinutes) # sets the start of query window + $poolEndTime = $endTime.AddMinutes(-$poolLagMinutes) # sets end of query window + + Write-Host "Starting to collect elastic pool telemetry for period" $poolStartTime "to" $poolEndTime "(UTC)" + + # Collect metrics for all elastic pools in this server. + $sql = ` + "SELECT subscription_guid = CAST ('$($SubscriptionId)' AS uniqueidentifier), resource_group_name = '$($ResourceGroupName)', server_name = '$($ServerName)', location = '$($Location)', elastic_pool_name + , end_time, elastic_pool_dtu_limit, avg_cpu_percent, avg_data_io_percent, avg_log_write_percent as avg_log_io_percent, max_worker_percent, max_session_percent + ,(SELECT Max(v) FROM (VALUES (avg_cpu_percent), (avg_data_io_percent), (avg_log_write_percent)) AS value(v)) AS avg_DTU_percent , avg_storage_percent, elastic_pool_storage_limit_mb FROM sys.elastic_pool_resource_stats + WHERE end_time > '$($poolStartTime)' and end_time <= '$($poolEndTime)';" + + $poolResult = Invoke-Sqlcmd -ServerInstance $sourceServerFullName -Database "master" -Username $ServerCred.UserName -Password $ServerCred.GetNetworkCredential().Password -Query $sql -ConnectionTimeout 120 -QueryTimeout 3600 + + if ($poolResult -ne $null) + { + #bulk copy the pool telemetry metrics to output database + $bulkCopy = new-object ("Data.SqlClient.SqlBulkCopy") $outputConnection + $bulkCopy.BulkCopyTimeout = 600 + $bulkCopy.DestinationTableName = "$poolResourceStatsTable"; + $bulkCopy.WriteToServer($poolResult); + + Write-Host "Elastic pool telemetry loaded for server" $ServerName + } + else + { + Write-Host "No elastic pool telemetry found for server" $ServerName "for period" $poolStartTime "to" $poolEndTime "(UTC)" + } + + # gather elastic database telemetry if requested + if ($IncludeDatabases) + { + # Get the list of current elastic databases + $sql =` + "select Name, elastic_pool_name from sys.databases as db + inner join sys.database_service_objectives as dbso on db.database_id = dbso.database_id + where (dbso.service_objective = 'ElasticPool') and db.Name != 'master'" + + $dbList = Invoke-Sqlcmd -ServerInstance $sourceServerFullName -Database "master" -Username $ServerCred.UserName -Password $ServerCred.GetNetworkCredential().Password -Query $sql -ConnectionTimeout 120 -QueryTimeout 3600 + + if ($dbList -ne $null) + { + Write-Host $dbList.Count "elastic databases found on server" $ServerName + + # Collect telemetry for each elastic database + foreach ($db in $dbList) + { + $sql= ` + "Declare @db_size float; + SELECT @db_size = SUM(reserved_page_count) * 8.0/1024/1024 FROM sys.dm_db_partition_stats + SELECT subscription_guid = CAST ('$($SubscriptionId)' AS uniqueidentifier), resource_group_name = '$($ResourceGroupName)',server_name = '$($ServerName)', location = '$($Location)', elastic_pool_name = '$($db.elastic_pool_name)' + , '$($db.Name)' as database_name, end_time, dtu_limit as database_dtu_limit, avg_cpu_percent, avg_data_io_percent, avg_log_write_percent as avg_log_io_percent, max_worker_percent, max_session_percent + ,(SELECT Max(v) FROM (VALUES (avg_cpu_percent), (avg_data_io_percent), (avg_log_write_percent)) AS value(v)) AS avg_DTU_percent ,@db_size as db_size FROM sys.dm_db_resource_stats + WHERE end_time > '$($startTime)' and end_time <= '$($endTime)';" + + $dbResult = Invoke-Sqlcmd -ServerInstance $SourceServerFullName -Database $db.Name -Username $ServerCred.UserName -Password $ServerCred.GetNetworkCredential().Password -Query $sql -ConnectionTimeout 120 -QueryTimeout 3600 + + if ($dbResult -ne $null) + { + #bulk copy the data to the telemetry database + $bulkCopy = new-object ("Data.SqlClient.SqlBulkCopy") $outputConnection + $bulkCopy.BulkCopyTimeout = 600 + $bulkCopy.DestinationTableName = "$dbResourceStatsTable"; + $bulkCopy.WriteToServer($dbResult); + + Write-Host "Telemetry loaded for elastic database" $db.Name + } + else + { + Write-Host "No telemetry found for elastic database " $db.Name "for period" $startTime "to" $endTime + } + } + } + else + { + $now = [DateTime]::UtcNow + + Write-Host "No elastic databases found for" $ServerName "when checking at" $now + } + } + + Write-Host "Finished collection for server" $ServerName "for period" $startTime "to" $endTime + + # set up time period for next collection + $startTime = $startTime.AddMinutes($interval) + $endTime = $endTime.AddMinutes($interval) + + # end if the next period doesn't start before the finish time + If ($startTime -ge $finishTime) {break} + + Write-Host "Sleeping until" $endTime "(UTC)" + + do + { + Start-Sleep 1 + } until (([DateTime]::UtcNow) -ge $endTime) + } +} \ No newline at end of file diff --git a/samples/manage/azure-sql-db-elastic-pools/PoolTelemetryJobRunner.ps1 b/samples/manage/azure-sql-db-elastic-pools/PoolTelemetryJobRunner.ps1 new file mode 100644 index 0000000000..207500ac42 --- /dev/null +++ b/samples/manage/azure-sql-db-elastic-pools/PoolTelemetryJobRunner.ps1 @@ -0,0 +1,190 @@ +# ---------------------------------------------------------------------------------- +# +# Copyright Microsoft Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------------- +# +# Powershell script for loading telemetry data from elastic pools and elastic database +# into a telemetry database. To be used in conjunction with PoolTelemetry.ps1, which +# should be installed in the same directory. +# +# This script should be customized as required (see <<<) to select one or more source +# servers, and will then spawn a telemetry gathering job for each server. Each job +# will use Load-PoolTelemetryForServer in PoolTelemetry.ps1 to load telemetry for all +# pools and elastic databases on the server. Each spawned job will run for an +# extended period in the background, waking up periodically to load more telemetry. +# The interval between telemetry gathering and the total duration can be controlled +# by parameters set in this script. +# +# Pool telemetry is loaded from the server master database, database telemetry is +# optionally loaded from each elastic database. This script assumes a common +# user name and password is used to connect to all source servers and databases. +# +#------------------------------------------------------------------------------ +# +## Prompt for Azure login +Login-AzureRMAccount + +# Set the Azure subscription, needed if your Microsoft account is associated with multiple subscriptions <<< *** +$AzureSubscriptionName = '' +$Subscription = Get-AzureRmSubscription -SubscriptionName $AzureSubscriptionName | Select-AzureRmSubscription + +$SubscriptionId = $Subscription.Subscription.SubscriptionId + +## Get SQL Server credentials. NOTE: same credential assumed for all source servers <<< *** +$sourceCred = Get-Credential -Message 'User name and password for source server' -UserName ''# add user name here +$outputServerCred = Get-Credential -Message 'User name and password for telemetry database' -UserName ''# add user name here + +## Resource group and server used for source server selection <<< *** +$resourceGroupName = '' # name of resource group containing the server from which telemetry will be gathered - see https://portal.azure.com +$serverName = '' # name of server from which telemetry will be gathered - e.g. "myappserver" + +## Telemetry database server and database name <<< *** +$outputServerName = '' # server name of telemetry database, like "telemetryserver" +$outputDatabaseName = '' # telemetry database name, like "ElasticPoolTelemetry" + +## Set server list checking <<< *** +$staticServerList = $false # set to $true if server list will not change during telemetry gathering period + +## Set telemetry collection timing <<< *** +$intervalMinutes = 15 # interval between telemetry collections used by spawned server job (15-30 mins suggested) +$durationMinutes = 60 # total duration for collection of telemetry and checking servers; 0 for one time execution, or a multiple of the interval. + +# Set to $true to gather 15 sec telemetry for elastic databases. Caution: large volumes of data may be returned. <<< *** +[bool]$includeDatabases = $false # or $False + +$now = [DateTime]::UtcNow +[DateTime]$startTime = $now # sets the start time for the first telemetry collection +[DateTime]$finishTime = $now.AddMinutes($DurationMinutes) # sets the overall finish time for a telemetry collection session + +Write-Host "Starting telemetry gathering for period" $startTime "to" $finishTime + +# Initialize the jobs list +$jobs = @{} + +# Check for the server(s) from which to gather telemetry. As servers may be added or removed during the telemetry gathering +# period the server list is re-evaluated periodically + +# The following identifies the servers to be analyzed and starts a job for each server to gather telemetry data +# In each iteration the latest server list is compared to the running jobs list and additional jobs started or +# current jobs stopped as required. + +while ($startTime -le $finishTime) +{ + Write-Host "Finding servers as at" $startTime "(UTC)" + + # initialize the servers list for each iteration + $servers = [ordered]@{} + + # Use or adapt one of the following queries as needed to select the source servers from which to load telemetry. <<< *** + + ## Get a specific server + #$server = Get-AzureRmSqlServer -ResourceGroupName $resourceGroupName -ServerName $serverName + #$servers.Add($server.ServerName, $server) + + ## Get all servers in a specific resource group + #$serverList = Get-AzureRmSqlServer -ResourceGroupName $resourceGroupName + + ## Get all resources of type server in the subscription + #$resourceList = Find-AzureRmResource -ResourceType microsoft.sql/servers + + ## Get all resources of type server in a specific region + $resourceList = Find-AzureRmResource -ResourceType microsoft.sql/servers -ODataQuery "(Location eq 'australiasoutheast' or Location eq 'australiaeast')" + + ## Get all resources of type server with common name pattern + #$resourceList = Find-AzureRmResource -ResourceType Microsoft.Sql/servers -ResourceNameContains '' + + # If selecting resources via a $resourceList convert to an equivalent $serverList + foreach ($resource in $resourceList) + { + $server = Get-AzureRmSqlServer -ResourceGroupName $resource.ResourceGroupName -ServerName $resource.Name + $servers.Add($server.ServerName, $server) + } + + Write-Host $servers.Count "servers found" + + $scriptPath = "$PSScriptRoot\PoolTelemetry.ps1" + $initScript = (Get-Command $scriptPath).ScriptBlock + + # For each server, start a job to collect telemetry. Set job name to the server name and put the job in $jobs + foreach($server in $servers.Values) + { + if ($jobs.Contains($server.ServerName) -eq $false) + { + $job = Start-Job -Name $server.ServerName -ScriptBlock { + param ($sp, $inc, $sub, $rgn, $sn, $loc, $sc, $osn, $odn, $osc, $im, $dm) + . $sp + Load-PoolTelemetryForServer -IncludeDatabases $inc ` + -SubscriptionId $sub -ResourceGroupName $rgn -ServerName $sn -Location $loc -ServerCred $sc -OutputServerName $osn -OutputDatabaseName $odn -OutputServerCred $osc -IntervalMinutes $im -DurationMinutes $dm} ` + -ArgumentList $scriptPath, $includeDatabases, $SubscriptionId, $server.ResourceGroupName, $server.ServerName, $server.Location, $sourceCred, $outputServerName, $outputDatabaseName, $outputServerCred, $intervalMinutes, $durationMinutes + + $jobs.Add($server.ServerName, $job) + + Write-Host "Job started for server" $server.ServerName + + # Following is useful for debugging changes to PoolTelemetry.ps1. Best used with a single server unless you set the duration to 0 + #. $scriptPath + #Load-PoolTelemetryForServer -IncludeDatabases $IncludeDatabases -SubscriptionId $SubscriptionId -ResourceGroupName $server.ResourceGroupName ` + # -ServerName $server.ServerName -Location $server.Location -ServerCred $sourceCred -OutputServerName $outputServerName ` + # -OutputDatabaseName $outputDatabaseName -OutputServerCred $outputServerCred -IntervalMinutes $intervalMinutes -DurationMinutes $durationMinutes + } + } + + # if server list doesn't change then no need to re-evaluate server list or manage the jobs. Already spawned jobs will continue to run. + if ($staticServerList) {break} + + # Stop previously started jobs if the server has been deleted + # For each started job in $jobs, check if the server is still in the most recent server list; if not, stop the job + foreach($job in $jobs.Values) + { + if ($servers.Contains($job.Name) -eq $false) + { + Write-Host "Server" $server.ServerName "no longer exists. Stopping its telemetry gathering job" + + Stop-Job $job.Name + + # Remove-Job + # leaving above commented-out allows use of Receive-Job to inspect the trace info emitted by the job + } + } + + # set up start time for next evaluation of the server list + $startTime = $startTime.AddMinutes($intervalMinutes) + + # sleep until next start time to ensure telemetry continues to be gathered + Write-Host "Sleeping until" $startTime "(UTC)" + + do + { + Start-Sleep 1 + + } until (([DateTime]::UtcNow) -ge $startTime) +} + +# For a static server list this runner script terminates after one pass and the jobs are left running +# Otherwise the jobs are stopped as they will at this point have concluded telemetry collection. +# In both cases Job trace output can be inspected with Receive-Job unless the jobs are removed. +# use Get-Job and Receive-Job [n] -Keep where n is the job number to see the output. + +if (-not $staticServerList) +{ + # gathering period is complete, now stop all the jobs + + Write-Host "Session complete, stopping all jobs" + + Stop-Job * + + #Remove-Job * + # leaving above commented-out allows use of Receive-Job to inspect the trace info emitted by the job + + get-job +} + diff --git a/samples/manage/azure-sql-db-elastic-pools/readme.md b/samples/manage/azure-sql-db-elastic-pools/readme.md new file mode 100644 index 0000000000..dc4e11ae50 --- /dev/null +++ b/samples/manage/azure-sql-db-elastic-pools/readme.md @@ -0,0 +1,77 @@ +# Sample name + +Briefly describe what the sample does. Describe the benefit of running this sample. + + + +![Alt text](/media/image-name.png "") + + +### Contents + +[About this sample](#about-this-sample)
+[Before you begin](#before-you-begin)
+[Run this sample](#run-this-sample)
+[Sample details](#sample-details)
+[Disclaimers](#disclaimers)
+[Related links](#related-links)
+ + + + +## About this sample + + +1. **Applies to:** SQL Server 2016 (or higher), Azure SQL Database, Azure SQL Data Warehouse, Parallel Data Warehouse +1. **Key features:*Elastic pools* +1. **Workload:** +1. **Programming Language:*PowerShell* +1. **Authors:*sriniachyara* +1. **Update history:*5/6/16* + + + +## Before you begin + +To run this sample, you need the following prerequisites. + +**Software prerequisites:** + + +1. Azure SQL Database +2. Azure PowerShell + +**Azure prerequisites:** + + +1. Permission to create an Azure SQL Database + + + +## Run this sample + + + +1. Open the ps in PowerShell. + +2. Fill in the values for Resource Manager name, server name, user name, password. + + + + +## Sample details + +Collects data from the listed server. + + + +## Disclaimers +The code included in this sample is not intended to be ... + + + +## Related Links + + +For more information, see these articles: +