itglue · jeremycolby · Mar 10, 2020
diff --git a/Cloudflare/CloudflareITGlue/CloudflareITGlue.psd1 b/Cloudflare/CloudflareITGlue/CloudflareITGlue.psd1
@@ -3,7 +3,7 @@
 #
 # Generated by: Jeremy Colby
 #
-# Generated on: 10/30/2018
+# Generated on: 06/27/2019
 #
 
 @{
@@ -12,8 +12,20 @@
     RootModule        = 'CloudflareITGlue.psm1'
 
     # Version number of this module.
-    ModuleVersion     = '1.1.0'
-
+    ModuleVersion     = '1.2.0'
+    #   1.0: Matching zones to ITGlue orgs via txt record mechanism
+    #   1.1: Matching zones to ITGlue orgs via Domain tracker + minor improvements
+    #   1.2: Full logging functionality
+    #        Cloudflare's zone file export format changed, modified to account for this
+    #           - Also, re Cloudflare zone file import/upload - it is normal for Cloudflare to show error when reading the SOA record from the zone file
+    #           - All other records are still imported normally, This happens with an unmodified exported zone files as well - SOA is a backend setting in Cloudflare
+    #        Files with the same name in ITGlue on a flex asset are not unique, revision history would only show the newest file,
+    #           - Zone files now have unique filename via utc timestamp, revision history keeps copies of each file
+    #        Running the sync command automatically creates the flex asset type if it does not exist
+    #        Related items tagging
+    #        Lowered Cloudflare request buffer
+    #        Formatting + minor improvements
+
     # Supported PSEditions
     # CompatiblePSEditions = @()
 
@@ -24,16 +36,16 @@
     Author            = 'Jeremy Colby'
 
     # Company or vendor of this module
-    CompanyName       = ''
+    # CompanyName       = ''
 
     # Copyright statement for this module
-    Copyright         = '(c) 2018 Jeremy Colby. All rights reserved.'
+    Copyright         = '(c) 2019 Jeremy Colby. All rights reserved.'
 
     # Description of the functionality provided by this module
-    # Description = ''
+    Description       = 'Sync Cloudflare DNS Zones to ITGlue Client Organizations as Flex Assets'
 
     # Minimum version of the Windows PowerShell engine required by this module
-    PowerShellVersion = '5.0' #New-TemporaryFile seems like only incompatiblity with 4.0
+    PowerShellVersion = '5.0' # New-TemporaryFile seems like only incompatiblity with 4.0
 
     # Name of the Windows PowerShell host required by this module
     # PowerShellHostName = ''
@@ -69,19 +81,16 @@
     NestedModules     = 'Private\CloudflareWebRequest.ps1',
     'Private\CloudflareZoneData.ps1',
     'Private\ITGlueWebRequest.ps1',
+    'Private\New-CloudflareITGlueFlexAssetType.ps1',
     'Public\CloudflareITGlueAPIAuth.ps1',
-    'Public\New-CloudflareITGlueFlexAssetType.ps1',
     'Public\Sync-CloudflareITGlueFlexibleAssets.ps1'
 
-
     # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export.
     FunctionsToExport = 'Add-CloudflareITGlueAPIAuth',
     'Get-CloudflareITGlueAPIAuth',
     'Remove-CloudflareITGlueAPIAuth',
-    'New-CloudflareITGlueFlexAssetType',
     'Sync-CloudflareITGlueFlexibleAssets'
 
-
     # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export.
     CmdletsToExport   = @()
 

diff --git a/Cloudflare/CloudflareITGlue/CloudflareITGlue.psm1 b/Cloudflare/CloudflareITGlue/CloudflareITGlue.psm1
@@ -8,4 +8,6 @@ if (Test-Path "$ModuleBase\$env:username.auth") {
     $Global:ITGlueAPIKey = ($Auth.ITGlueAPIKey | ConvertTo-SecureString)
 }
 
+$Global:CFITGLog = $null
+
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
diff --git a/Cloudflare/CloudflareITGlue/Private/CloudflareWebRequest.ps1 b/Cloudflare/CloudflareITGlue/Private/CloudflareWebRequest.ps1
@@ -12,13 +12,18 @@ function New-CloudflareWebRequest {
             $APIKey = [System.Runtime.InteropServices.Marshal]::PtrToStringAuto([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($CloudflareAPIKey))
         }
         catch {
-            Write-Warning 'New-CloudflareWebRequest:  Unable to decrypt auth info'
-            Write-Warning 'Run Add-CloudflareITGlueAPIAuth to re-add'
+            Write-Warning 'Unable to decrypt auth info, run Add-CloudflareITGlueAPIAuth to re-add'
+            if ($CFITGLog) {
+                "[CF Request]$(Get-Date -Format G):  Unable to decrypt auth info, run Add-CloudflareITGlueAPIAuth to re-add" | Out-File $CFITGLog -Append
+            }
             break
         }
     }
     else {
         Write-Warning 'Run Add-CloudflareITGlueAPIAuth to add authorization info'
+        if ($CFITGLog) {
+            "[CF Request]$(Get-Date -Format G):  Run Add-CloudflareITGlueAPIAuth to add authorization info" | Out-File $CFITGLog -Append
+        }
         break
     }
 
@@ -31,13 +36,12 @@ function New-CloudflareWebRequest {
             'Content-Type' = 'application/json'
         }
     }
-    if ($Body) {$RequestParams.Body = $Body}
+    if ($Body) { $RequestParams.Body = $Body }
 
     try {
-        Start-Sleep -Milliseconds 275
         $Request = Invoke-RestMethod @RequestParams
-        Start-Sleep -Milliseconds 275
-        #RateLimit = 1200 requests/5 min but still getting gateway timeouts @ 300ms+
+        Start-Sleep -Milliseconds 325
+        # RateLimit: 1200/5 min
 
         if ($PageNumber -lt $Request.result_info.total_pages) {
             $PageNumber++
@@ -49,6 +53,10 @@ function New-CloudflareWebRequest {
     }
     catch {
         Write-Warning "Something went wrong with Cloudflare request:`n$_"
+        if ($CFITGLog) {
+            "[CF Request: $Endpoint]$(Get-Date -Format G):  $_" | Out-File $CFITGLog -Append
+        }
+
         $APIKey = $null
         $RequestParams = $null
     }

diff --git a/Cloudflare/CloudflareITGlue/Private/CloudflareZoneData.ps1 b/Cloudflare/CloudflareITGlue/Private/CloudflareZoneData.ps1
@@ -4,41 +4,47 @@ function Get-CloudflareZoneData {
         [Parameter(Mandatory = $true)][pscustomobject]$ITGMatch
     )
 
-    $Timestamp = (Get-Date).ToUniversalTime().ToString("yyyy-M-d HH:mm:ss")
+    $DateUTC = (Get-Date).ToUniversalTime().ToString('yyyy-M-d')
     $AccountId = New-CloudflareWebRequest -Endpoint 'accounts' | ForEach-Object result | ForEach-Object id
     $ZoneInfo = New-CloudflareWebRequest -Endpoint "zones/$ZoneId"
     $ZoneRecords = New-CloudflareWebRequest -Endpoint "zones/$ZoneId/dns_records"
     if ($ZoneRecords.result_info.count -eq 0) {
         Write-Host "$($ZoneInfo.result.name): Empty Zone Detected" -ForegroundColor Yellow
+        if ($CFITGLog) {
+            "[CF]$(Get-Date -Format G):  Empty Zone Detected - $($ZoneInfo.result.name)" | Out-File $CFITGLog -Append
+        }
         break
     }
     $ZoneFileData = New-CloudflareWebRequest -Endpoint "zones/$ZoneId/dns_records/export"
     $ZoneFileData = $ZoneFileData.Replace(
-        "@	3600	IN	SOA	$($ZoneInfo.result.name).	root.$($ZoneInfo.result.name).	(",
-        "@	3600	IN	SOA	$($ZoneInfo.result.name_servers[0]). $((($CloudflareAPIEmail -split '@')[0]).Replace('.','\.') + '.'+ ($CloudflareAPIEmail -split '@')[1]). ("
+        ';; SOA Record',
+        "`$ORIGIN $($ZoneInfo.result.name).`n`n;; SOA Record"
+    )
+    $ZoneFileData = $ZoneFileData.Replace(
+        "SOA`t$($ZoneInfo.result.name). root.$($ZoneInfo.result.name).",
+        "SOA`t$($ZoneInfo.result.name_servers[0]). $((($CloudflareAPIEmail -split '@')[0]).Replace('.','\.') + '.'+ ($CloudflareAPIEmail -split '@')[1])."
     )
     $ZoneFileData = $ZoneFileData.Replace(
-        ";; NS Records (YOU MUST CHANGE THIS)`n$($ZoneInfo.result.name).	1	IN	NS	" + 'REPLACE&ME$WITH^YOUR@NAMESERVER.',
-        ";; NS Records`n$($ZoneInfo.result.name).	1	IN	NS	$($ZoneInfo.result.name_servers[0]).`n$($ZoneInfo.result.name).	1	IN	NS	$($ZoneInfo.result.name_servers[1])."
+        ';; A Records',
+        ";; NS Records`n$($ZoneInfo.result.name).	1	IN	NS	$($ZoneInfo.result.name_servers[0]).`n$($ZoneInfo.result.name).	1	IN	NS	$($ZoneInfo.result.name_servers[1]).`n`n;; A Records"
     )
     $ZoneFileData = $ZoneFileData.Replace(
         ';;   -- update the SOA record with the correct authoritative name server',
-        ";;   -- update the SOA record with the correct authoritative name server`n;;   ** CloudflareITGlue Module: Updated $($Timestamp)"
+        ";;   -- update the SOA record with the correct authoritative name server`n;;   ** CloudflareITGlue Module: Updated $($DateUTC)"
     )
     $ZoneFileData = $ZoneFileData.Replace(
         ';;   -- update the SOA record with the contact e-mail address information',
-        ";;   -- update the SOA record with the contact e-mail address information`n;;   ** CloudflareITGlue Module: Updated $($Timestamp)"
+        ";;   -- update the SOA record with the contact e-mail address information`n;;   ** CloudflareITGlue Module: Updated $($DateUTC)"
     )
     $ZoneFileData = $ZoneFileData.Replace(
         ';;   -- update the NS record(s) with the authoritative name servers for this domain.',
-        ";;   -- update the NS record(s) with the authoritative name servers for this domain.`n;;   ** CloudflareITGlue Module: Updated $($Timestamp)"
+        ";;   -- update the NS record(s) with the authoritative name servers for this domain.`n;;   ** CloudflareITGlue Module: Updated $($DateUTC)`n;;   ** CloudflareITGlue Module: Added `$ORIGIN directive"
     )
     $RecordsHtml = 
     '<div>
         <p><a class="btn btn-sm btn-default" href="https://dash.cloudflare.com/' + $AccountId + "/$($ZoneInfo.result.name)" + '/dns" rel="nofollow" style="display: inline" title="Cloudflare">
-            <i class="fa fa-fw fa-external-link"></i>
-            <span>Open in Cloudflare</span>
-        </a></p>
+        <i class="fa fa-fw fa-external-link"></i><span>Open in Cloudflare</span></a></p>
+
         <table id="RecordTable" style="width:100%">
             <thead>
                 <th>Type</th>
@@ -50,24 +56,24 @@ function Get-CloudflareZoneData {
                 <th>Modified</th>
             </thead>
             <tbody>' +
-            $(foreach ($Record in $ZoneRecords.result) {
-                "<tr>
-                    <td>$($Record.type)</td>
-                    <td>$($Record.name)</td>
-                    <td>$($Record.content)</td>
-                    <td>$($Record.priority)</td>
-                    <td>$(if ($Record.ttl -eq 1){'Auto'}else{$Record.ttl})</td>
-                    <td>$($Record.proxied)</td>
-                    <td>$(($Record.modified_on.Replace('T', ' ') -split '\.')[0])</td>
-                </tr>"
-            }) +
+                $(foreach ($Record in $ZoneRecords.result) {
+                    "<tr>
+                        <td>$($Record.type)</td>
+                        <td>$($Record.name)</td>
+                        <td>$($Record.content)</td>
+                        <td>$($Record.priority)</td>
+                        <td>$(if ($Record.ttl -eq 1){'Auto'}else{$Record.ttl})</td>
+                        <td>$($Record.proxied)</td>
+                        <td>$(($Record.modified_on.Replace('T', ' ') -split '\.')[0])</td>
+                    </tr>"
+                }) +
             '</tbody>
         </table>
     </div>'
 
     $ZoneData = [ordered]@{
         Name          = $ZoneInfo.result.name
-        SyncDate      = $Timestamp
+        SyncDate      = $DateUTC
         CfNameServers = $ZoneInfo.result.name_servers
         Status        = $ZoneInfo.result.status
         ZoneFileData  = $ZoneFileData
@@ -79,11 +85,12 @@ function Get-CloudflareZoneData {
 }
 
 function Get-CloudflareZoneDataArray {
+    $Progress = 0
+    Write-Progress -Activity 'CloudflareAPI' -Status 'Getting Zone Data' -PercentComplete 0 -Id 1
     $ZoneDataArray = @()
     $AllZones = New-CloudflareWebRequest -Endpoint 'zones'
     [pscustomobject]$ITGDomains = New-ITGlueWebRequest -Endpoint 'domains' | ForEach-Object data
-    $Progress = 0
-
+
     foreach ($Zone in $AllZones.result) {
         Write-Progress -Activity 'CloudflareAPI' -Status 'Getting Zone Data' -CurrentOperation $Zone.name -PercentComplete ($Progress / ($AllZones.result | Measure-Object | ForEach-Object count) * 100) -Id 1
 
@@ -97,16 +104,19 @@ function Get-CloudflareZoneDataArray {
                 $ITGMatches += [pscustomobject]$Match
             }
         }
-        if($ITGMatches){
+        if ($ITGMatches) {
             foreach ($Match in $ITGMatches) {
                 $ZoneData = Get-CloudflareZoneData -ZoneId $Zone.id -ITGMatch $Match
                 if ($ZoneData) {
                     $ZoneDataArray += [pscustomobject]$ZoneData
                 }
             }
         }
-        else{
+        else {
             Write-Host "$($Zone.name): Add to domain tracker" -ForegroundColor Yellow
+            if ($CFITGLog) {
+                "[CFITG]$(Get-Date -Format G):  $($Zone.name) - Add to ITG domain tracker" | Out-File $CFITGLog -Append
+            }
         }
         $Progress++
     }

diff --git a/Cloudflare/CloudflareITGlue/Private/ITGlueWebRequest.ps1 b/Cloudflare/CloudflareITGlue/Private/ITGlueWebRequest.ps1
@@ -4,21 +4,27 @@ function New-ITGlueWebRequest {
         [ValidateSet('GET', 'POST', 'PATCH', 'DELETE')][string]$Method = 'GET',
         [string]$Body = $null,
         [int]$ResultsPerPage = 50,
-        [int]$PageNumber = 1
+        [int]$PageNumber = 1,
+        [string]$Params = $null
     )
 
     if ($ITGlueAPIKey) {
         try {
             $APIKey = [System.Runtime.InteropServices.Marshal]::PtrToStringAuto([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($ITGlueAPIKey))
         }
         catch {
-            Write-Warning 'New-ITGlueWebRequest:  Unable to decrypt auth info'
-            Write-Warning 'Run Add-CloudflareITGlueAPIAuth to re-add'
+            Write-Warning 'Unable to decrypt auth info, run Add-CloudflareITGlueAPIAuth to re-add'
+            if ($CFITGLog) {
+                "[ITG Request]$(Get-Date -Format G):  Unable to decrypt auth info, run Add-CloudflareITGlueAPIAuth to re-add" | Out-File $CFITGLog -Append
+            }
             break
         }
     }
     else {
         Write-Warning 'Run Add-CloudflareITGlueAPIAuth to add authorization info'
+        if ($CFITGLog) {
+            "[ITG Request]$(Get-Date -Format G):  Run Add-CloudflareITGlueAPIAuth to add authorization info" | Out-File $CFITGLog -Append
+        }
         break
     }
 
@@ -30,10 +36,12 @@ function New-ITGlueWebRequest {
             'Content-Type' = 'application/vnd.api+json'
         }
     }
-    if ($Body) {$RequestParams.Body = $Body}
+    if ($Params) { $RequestParams.Uri += "&$Params" }
+    if ($Body) { $RequestParams.Body = $Body }
 
     try {
         $Request = Invoke-RestMethod @RequestParams
+        # RateLimit: 10k/day
 
         if ($PageNumber -lt $Request.meta.'total-pages') {
             $PageNumber++
@@ -45,6 +53,10 @@ function New-ITGlueWebRequest {
     }
     catch {
         Write-Warning "Something went wrong with ITGlue request:`n$_"
+        if ($CFITGLog) {
+            "[ITG Request: $Endpoint]$(Get-Date -Format G):  $_" | Out-File $CFITGLog -Append
+        }
+
         $APIKey = $null
         $RequestParams = $null
     }

diff --git a/...lic/New-CloudflareITGlueFlexAssetType.ps1 → ...ate/New-CloudflareITGlueFlexAssetType.ps1 b/...lic/New-CloudflareITGlueFlexAssetType.ps1 → ...ate/New-CloudflareITGlueFlexAssetType.ps1
@@ -68,7 +68,7 @@ function New-CloudflareITGlueFlexAssetType {
                                 order        = 5
                                 name         = 'Zone File'
                                 kind         = 'Upload'
-                                hint         = 'Exported zone file in BIND format. You can upload this to Cloudflare. UTF-8 Encoded, use notepad++ for better viewing.'
+                                hint         = 'Exported zone file in BIND format. You can upload this to Cloudflare.'
                                 required     = $false
                                 show_in_list = $false
                             }
@@ -77,18 +77,6 @@ function New-CloudflareITGlueFlexAssetType {
                             type       = 'flexible_asset_fields'
                             attributes = @{
                                 order        = 6
-                                name         = 'Domain Tracker'
-                                kind         = 'Tag'
-                                hint         = 'Tagged in Domain Tracker'
-                                tag_type     = 'Domains'
-                                required     = $false
-                                show_in_list = $false
-                            }
-                        },
-                        @{
-                            type       = 'flexible_asset_fields'
-                            attributes = @{
-                                order        = 7
                                 name         = 'DNS Records'
                                 kind         = 'Textbox'
                                 hint         = 'Table of DNS records in the zone'
@@ -103,4 +91,4 @@ function New-CloudflareITGlueFlexAssetType {
     }
     $Body = $Body | ConvertTo-Json -Depth 6
     New-ITGlueWebRequest -Endpoint 'flexible_asset_types' -Method 'POST' -Body $Body
-}
+}