feat: Support deletion of Billing Account Log Sinks (#227)

Author: daniel-cit

modules/project_cleanup/README.md

@@ -12,34 +12,40 @@ Running this module requires an App Engine app in the specified project/region.
 
 The following services must be enabled on the project housing the cleanup function prior to invoking this module:
 
+- Artifact Registry API (`artifactregistry.googleapis.com`)
 - Cloud Functions (`cloudfunctions.googleapis.com`)
 - Cloud Scheduler (`cloudscheduler.googleapis.com`)
 - Cloud Resource Manager (`cloudresourcemanager.googleapis.com`)
 - Compute Engine API (`compute.googleapis.com`)
 - Cloud Asset API (`cloudasset.googleapis.com`)
 - Security Command Center API (`securitycenter.googleapis.com`)
+- Cloud Logging API (`logging.googleapis.com`)
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| billing\_account | Billing Account used to provision resources. | `string` | `""` | no |
+| clean\_up\_billing\_sinks | Clean up Billing Account Sinks. | `bool` | `false` | no |
 | clean\_up\_org\_level\_cai\_feeds | Clean up organization level Cloud Asset Inventory Feeds. | `bool` | `false` | no |
 | clean\_up\_org\_level\_scc\_notifications | Clean up organization level Security Command Center notifications. | `bool` | `false` | no |
 | clean\_up\_org\_level\_tag\_keys | Clean up organization level Tag Keys. | `bool` | `false` | no |
 | function\_timeout\_s | The amount of time in seconds allotted for the execution of the function. | `number` | `500` | no |
 | job\_schedule | Cleaner function run frequency, in cron syntax | `string` | `"*/5 * * * *"` | no |
+| list\_billing\_sinks\_page\_size | The maximum number of Billing Account Log Sinks to return in the call to `BillingAccountsSinksService.List` service. | `number` | `200` | no |
 | list\_scc\_notifications\_page\_size | The maximum number of notification configs to return in the call to `ListNotificationConfigs` service. The minimun value is 1 and the maximum value is 1000. | `number` | `500` | no |
 | max\_project\_age\_in\_hours | The maximum number of hours that a GCP project, selected by `target_tag_name` and `target_tag_value`, can exist | `number` | `6` | no |
 | organization\_id | The organization ID whose projects to clean up | `string` | n/a | yes |
 | project\_id | The project ID to host the scheduled function in | `string` | n/a | yes |
 | region | The region the project is in (App Engine specific) | `string` | n/a | yes |
+| target\_billing\_sinks | List of Billing Account Log Sinks names regex that will be deleted. Regex example: `.*/sinks/sk-c-logging-.*-billing-.*` | `list(string)` | `[]` | no |
 | target\_excluded\_labels | Map of project lablels that won't be deleted. | `map(string)` | `{}` | no |
 | target\_excluded\_tagkeys | List of organization Tag Key short names that won't be deleted. | `list(string)` | `[]` | no |
 | target\_folder\_id | Folder ID to delete all projects under. | `string` | `""` | no |
 | target\_included\_feeds | List of organization level Cloud Asset Inventory feeds that should be deleted. Regex example: `.*/feeds/fd-cai-monitoring-.*` | `list(string)` | `[]` | no |
 | target\_included\_labels | Map of project lablels that will be deleted. | `map(string)` | `{}` | no |
-| target\_included\_scc\_notifications | List of organization  Security Command Center notifications names regex that will be deleted. Regex example: `.*/notificationConfigs/scc-notify-.*` | `list(string)` | `[]` | no |
+| target\_included\_scc\_notifications | List of organization Security Command Center notifications names regex that will be deleted. Regex example: `.*/notificationConfigs/scc-notify-.*` | `list(string)` | `[]` | no |
 | target\_tag\_name | The name of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead). | `string` | `""` | no |
 | target\_tag\_value | The value of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead). | `string` | `""` | no |
 | topic\_name | Name of pubsub topic connecting the scheduled projects cleanup function | `string` | `"pubsub_scheduled_project_cleaner"` | no |

modules/project_cleanup/function_source/README.md

@@ -15,11 +15,24 @@ The following environment variables may be specified to configure the cleanup ut
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
+| `BILLING_ACCOUNT` | Billing Account used to provision resources. | `string` | n/a | no |
+| `BILLING_SINKS_PAGE_SIZE ` | The maximum number of Billing Account Log Sinks to return in the call to `BillingAccountsSinksService.List` service. | `number` | n/a | yes |
+| `CLEAN_UP_BILLING_SINKS` | Clean up Billing Account Sinks. | `bool` | n/a | yes |
+| `CLEAN_UP_CAI_FEEDS`| Clean up organization level Cloud Asset Inventory Feeds. | `bool` | n/a | yes |
+| `CLEAN_UP_SCC_NOTIFICATIONS` | Clean up organization level Security Command Center notifications. | `bool` | n/a | yes |
+| `CLEAN_UP_TAG_KEYS` | Clean up organization level Tag Keys. | `bool` | n/a | yes |
+| `MAX_PROJECT_AGE_HOURS` | The project age, in hours, at which point deletion should be considered | integer | n/a | yes |
+| `SCC_NOTIFICATIONS_PAGE_SIZE` | The maximum number of notification configs to return in the call to `ListNotificationConfigs` service. The minimun value is 1 and the maximum value is 1000. | `number` | n/a | yes |
+| `TARGET_BILLING_SINKS` | List of Billing Account Log Sinks names regex that will be deleted. Regex example: `.*/sinks/sk-c-logging-.*-billing-.*` | `list(string)` | n/a | no |
 | `TARGET_EXCLUDED_LABELS` | Labels to match on for identifying projects to avoid deletion | string | n/a | no |
+| `TARGET_EXCLUDED_TAGKEYS` | List of organization Tag Key short names that won't be deleted. | `list(string)` | n/a | no |
 | `TARGET_FOLDER_ID` | Folder ID to delete projects under | string | n/a | yes |
+| `TARGET_INCLUDED_FEEDS` | List of organization level Cloud Asset Inventory feeds that should be deleted. Regex example: `.*/feeds/fd-cai-monitoring-.*` | `list(string)` | n/a | no |
 | `TARGET_INCLUDED_LABELS` | Labels to match on for identifying projects to delete | string | n/a | no |
-| `MAX_PROJECT_AGE_HOURS` | The project age, in hours, at which point deletion should be considered | integer | n/a | yes |
+| `TARGET_INCLUDED_SCC_NOTIFICATIONS` | List of organization Security Command Center notifications names regex that will be deleted. Regex example: `.*/notificationConfigs/scc-notify-.*` | `list(string)` | n/a | no |
+| `TARGET_ORGANIZATION_ID` | The organization ID whose projects to clean up | `string` | n/a | yes |
 
 ## Required Permissions
 
 This Cloud Function must be run as a Service Account with the `Organization Administrator` (`roles/resourcemanager.organizationAdmin`) role.
+If `CLEAN_UP_BILLING_SINKS` is enabled the Service Account running the Cloud Function needs role Logs Configuration Writer(`roles/logging.configWriter`) in the billing account `BILLING_ACCOUNT`.

modules/project_cleanup/function_source/main.go

@@ -39,6 +39,7 @@ import (
 	"google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
+	"google.golang.org/api/logging/v2"
 	"google.golang.org/api/option"
 	"google.golang.org/api/servicemanagement/v1"
 )
@@ -56,25 +57,34 @@ const (
 	MaxProjectAgeHours            = "MAX_PROJECT_AGE_HOURS"
 	targetFolderRegexp            = `^[0-9]+$`
 	targetOrganizationRegexp      = `^[0-9]+$`
+	billingAccountRegex           = `^[0-9A-Z][-0-9A-Z]{18}[0-9A-Z]$`
 	SCCNotificationsPageSize      = "SCC_NOTIFICATIONS_PAGE_SIZE"
 	CleanUpCaiFeeds               = "CLEAN_UP_CAI_FEEDS"
 	TargetIncludedFeeds           = "TARGET_INCLUDED_FEEDS"
+	BillingAccount                = "BILLING_ACCOUNT"
+	CleanUpBillingSinks           = "CLEAN_UP_BILLING_SINKS"
+	TargetBillingSinks            = "TARGET_BILLING_SINKS"
+	BillingSinksPageSize          = "BILLING_SINKS_PAGE_SIZE"
 )
 
 var (
 	logger                 = log.New(os.Stdout, "", 0)
 	excludedLabelsMap      = getLabelsMapFromEnv(TargetExcludedLabels)
 	includedLabelsMap      = getLabelsMapFromEnv(TargetIncludedLabels)
-	cleanUpTagKeys         = getCleanUpTagKeysOrTerminateExecution()
-	cleanUpSCCNotfi        = getCleanUpSCCNotfiOrTerminateExecution()
+	cleanUpTagKeys         = getBoolFromEnv(CleanUpTagKeys)
+	cleanUpSCCNotfi        = getBoolFromEnv(CleanUpSCCNotfi)
 	excludedTagKeysList    = getTagKeysListFromEnv(TargetExcludedTagKeys)
-	includedSCCNotfisList  = getSCCNotfiListFromEnv(TargetIncludedSCCNotfis)
-	resourceCreationCutoff = getOldTime(int64(getCorrectMaxAgeInHoursOrTerminateExecution()) * 60 * 60)
+	includedSCCNotfisList  = getRegexListFromEnv(TargetIncludedSCCNotfis)
+	resourceCreationCutoff = getOldTime(getIntFromEnv(MaxProjectAgeHours) * 60 * 60)
 	rootFolderId           = getCorrectFolderIdOrTerminateExecution()
 	organizationId         = getCorrectOrganizationIdOrTerminateExecution()
-	sccPageSize            = getSCCNotificationPageSizeOrTerminateExecution()
-	cleanUpCaiFeeds        = getCleanUpFeedsOrTerminateExecution()
-	includedFeedsList      = getFeedsListFromEnv(TargetIncludedFeeds)
+	sccPageSize            = int32(getIntFromEnv(SCCNotificationsPageSize))
+	cleanUpCaiFeeds        = getBoolFromEnv(CleanUpCaiFeeds)
+	includedFeedsList      = getRegexListFromEnv(TargetIncludedFeeds)
+	billingAccount         = getBillingAccountOrTerminateExecution()
+	cleanUpBillingSinks    = getBoolFromEnv(CleanUpBillingSinks)
+	billingSinksPageSize   = getIntFromEnv(BillingSinksPageSize)
+	targetBillingSinks     = getRegexListFromEnv(TargetBillingSinks)
 )
 
 type PubSubMessage struct {
@@ -155,15 +165,6 @@ func processProjectsResponsePage(removeProjectById func(projectId string)) func(
 	}
 }
 
-func getCorrectMaxAgeInHoursOrTerminateExecution() int64 {
-	maxAgeInHoursStr := os.Getenv(MaxProjectAgeHours)
-	maxAgeInHours, err := strconv.ParseInt(os.Getenv(MaxProjectAgeHours), 10, 0)
-	if err != nil {
-		logger.Fatalf("Could not convert [%s] to integer. Specify correct value for environment variable [%s] and try again.", maxAgeInHoursStr, MaxProjectAgeHours)
-	}
-	return maxAgeInHours
-}
-
 func checkIfAtLeastOneLabelPresentIfAny(project *cloudresourcemanager.Project, labels map[string]string, isExcludeCheck bool) bool {
 	if len(labels) == 0 {
 		return !isExcludeCheck
@@ -221,29 +222,29 @@ func getLabelsMapFromEnv(envVariableName string) map[string]string {
 	return labels
 }
 
-func getSCCNotfiListFromEnv(envVariableName string) []*regexp.Regexp {
+func getRegexListFromEnv(envVariableName string) []*regexp.Regexp {
 	var compiledRegEx []*regexp.Regexp
-	targetExcludedSCCNotfis := os.Getenv(envVariableName)
-	logger.Println("Try to get SCC Notifications list")
-	if targetExcludedSCCNotfis == "" {
-		logger.Printf("No SCC Notifications provided.")
+	envListVar := os.Getenv(envVariableName)
+	logger.Printf("Try to get [%s] list", envVariableName)
+	if envListVar == "" {
+		logger.Printf("No value for [%s] list provided.", envVariableName)
 		return compiledRegEx
 	}
 
-	var sccNotfis []string
-	err := json.Unmarshal([]byte(targetExcludedSCCNotfis), &sccNotfis)
+	var regexList []string
+	err := json.Unmarshal([]byte(envListVar), &regexList)
 	if err != nil {
-		logger.Printf("Failed to get SCC Notifications list from [%s] env variable, error [%s]", envVariableName, err.Error())
+		logger.Printf("Failed to get Regex list from [%s] env variable, error [%s]", envVariableName, err.Error())
 		return compiledRegEx
 	} else {
-		logger.Printf("Got SCC Notifications list [%s] from [%s] env variable", sccNotfis, envVariableName)
+		logger.Printf("Got Regex list [%s] from [%s] env variable", regexList, envVariableName)
 	}
 
 	//build Regexes
-	for _, r := range sccNotfis {
+	for _, r := range regexList {
 		result, err := regexp.Compile(r)
 		if err != nil {
-			logger.Printf("Invalid regular expression [%s] for SCC Notification", r)
+			logger.Printf("Invalid regular expression [%s] for [%s]", r, envVariableName)
 		} else {
 			compiledRegEx = append(compiledRegEx, result)
 		}
@@ -269,79 +270,25 @@ func getTagKeysListFromEnv(envVariableName string) []string {
 	return tagKeys
 }
 
-func getCleanUpTagKeysOrTerminateExecution() bool {
-	cleanUpTagKeys, exists := os.LookupEnv(CleanUpTagKeys)
-	if !exists {
-		logger.Fatalf("Clean up Tag Keys environment variable [%s] not set, set the environment variable and try again.", CleanUpTagKeys)
-	}
-	result, err := strconv.ParseBool(cleanUpTagKeys)
-	if err != nil {
-		logger.Fatalf("Invalid Clean up Tag Keys value [%s], specify correct value for environment variable [%s] and try again.", cleanUpTagKeys, CleanUpTagKeys)
-	}
-	return result
-}
-
-func getCleanUpSCCNotfiOrTerminateExecution() bool {
-	cleanUpSCCNotfiVal, exists := os.LookupEnv(CleanUpSCCNotfi)
+func getBoolFromEnv(envVariableName string) bool {
+	envVariableNameVal, exists := os.LookupEnv(envVariableName)
 	if !exists {
-		logger.Fatalf("Clean up SCC notifications environment variable [%s] not set, set the environment variable and try again.", CleanUpSCCNotfi)
+		logger.Fatalf("Environment variable [%s] not set, set the environment variable and try again.", envVariableName)
 	}
-	result, err := strconv.ParseBool(cleanUpSCCNotfiVal)
+	result, err := strconv.ParseBool(envVariableNameVal)
 	if err != nil {
-		logger.Fatalf("Invalid Clean up SCC notifications value [%s], specify correct value for environment variable [%s] and try again.", cleanUpSCCNotfiVal, CleanUpSCCNotfi)
+		logger.Fatalf("Invalid bool value [%s], specify correct value for environment variable [%s] and try again.", envVariableNameVal, envVariableName)
 	}
 	return result
 }
 
-func getSCCNotificationPageSizeOrTerminateExecution() int32 {
-	pageSize := os.Getenv(SCCNotificationsPageSize)
-	size, err := strconv.ParseInt(pageSize, 10, 32)
-	if err != nil {
-		logger.Fatalf("Invalid page size [%s], specify correct value and try again.", pageSize)
-	}
-	return int32(size)
-}
-
-func getFeedsListFromEnv(envVariableName string) []*regexp.Regexp {
-	var compiledRegEx []*regexp.Regexp
-	targetIncludedFeeds := os.Getenv(envVariableName)
-	logger.Println("Try to get CAI Feeds list")
-	if targetIncludedFeeds == "" {
-		logger.Printf("No CAI Feeds provided.")
-		return compiledRegEx
-	}
-
-	var caiFeeds []string
-	err := json.Unmarshal([]byte(targetIncludedFeeds), &caiFeeds)
+func getIntFromEnv(envVariableName string) int64 {
+	envVariableStr := os.Getenv(envVariableName)
+	intValue, err := strconv.ParseInt(envVariableStr, 10, 0)
 	if err != nil {
-		logger.Printf("Failed to get CAI Feeds list from [%s] env variable, error [%s]", envVariableName, err.Error())
-		return nil
-	} else {
-		logger.Printf("Got CAI Feeds list [%s] from [%s] env variable", caiFeeds, envVariableName)
-	}
-
-	//build Regexes
-	for _, r := range caiFeeds {
-		result, err := regexp.Compile(r)
-		if err != nil {
-			logger.Printf("Invalid regular expression [%s] for CAI Feed", r)
-		} else {
-			compiledRegEx = append(compiledRegEx, result)
-		}
+		logger.Fatalf("Could not convert [%s] to integer. Specify correct value for environment variable [%s] and try again.", envVariableStr, envVariableName)
 	}
-	return compiledRegEx
-}
-
-func getCleanUpFeedsOrTerminateExecution() bool {
-	cleanUpCaiFeeds, exists := os.LookupEnv(CleanUpCaiFeeds)
-	if !exists {
-		logger.Fatalf("Clean up CAI Feeds environment variable [%s] not set, set the environment variable and try again.", CleanUpCaiFeeds)
-	}
-	result, err := strconv.ParseBool(cleanUpCaiFeeds)
-	if err != nil {
-		logger.Fatalf("Invalid Clean up CAI Feeds value [%s], specify correct value for environment variable [%s] and try again.", cleanUpCaiFeeds, CleanUpCaiFeeds)
-	}
-	return result
+	return intValue
 }
 
 func getCorrectFolderIdOrTerminateExecution() string {
@@ -353,6 +300,21 @@ func getCorrectFolderIdOrTerminateExecution() string {
 	return targetFolderIdString
 }
 
+func getBillingAccountOrTerminateExecution() string {
+	billingAccountVal := os.Getenv(BillingAccount)
+	if billingAccountVal == "" {
+		if cleanUpBillingSinks {
+			logger.Fatal("If billing account sink clean up is enabled, billing account id should not be empty, specify correct value and try again.")
+		}
+		return billingAccountVal
+	}
+	matched, err := regexp.MatchString(billingAccountRegex, billingAccountVal)
+	if err != nil || !matched {
+		logger.Fatalf("Invalid billing account id [%s], specify correct value and try again.", billingAccountVal)
+	}
+	return billingAccountVal
+}
+
 func getCorrectOrganizationIdOrTerminateExecution() string {
 	targetOrganizationIdString := os.Getenv(TargetOrganizationId)
 	matched, err := regexp.MatchString(targetOrganizationRegexp, targetOrganizationIdString)
@@ -410,6 +372,15 @@ func getTagValuesServiceOrTerminateExecution(ctx context.Context, client *http.C
 	return cloudResourceManagerService.TagValues
 }
 
+func getBillingAccountSinkServiceOrTerminateExecution(ctx context.Context, client *http.Client) *logging.BillingAccountsSinksService {
+	loggingService, err := logging.NewService(ctx, option.WithHTTPClient(client))
+	if err != nil {
+		logger.Fatalf("Failed to get Logging Sink Service with error [%s], terminate execution", err.Error())
+	}
+	logger.Println("Got Logging Sink Service")
+	return loggingService.BillingAccounts.Sinks
+}
+
 func getSCCNotificationServiceOrTerminateExecution(ctx context.Context, client *http.Client) *securitycenter.Client {
 	logger.Println("Try to get SCC Notification Service")
 	securitycenterClient, err := securitycenter.NewClient(ctx)
@@ -458,6 +429,7 @@ func invoke(ctx context.Context) {
 	sccService := getSCCNotificationServiceOrTerminateExecution(ctx, client)
 	tagValuesService := getTagValuesServiceOrTerminateExecution(ctx, client)
 	feedsService := getAssetServiceOrTerminateExecution(ctx, client)
+	billingSinkService := getBillingAccountSinkServiceOrTerminateExecution(ctx, client)
 	firewallPoliciesService := getFirewallPoliciesServiceOrTerminateExecution(ctx, client)
 	endpointService := getServiceManagementServiceOrTerminateExecution(ctx, client)
 
@@ -480,6 +452,15 @@ func invoke(ctx context.Context) {
 		return tagKeyCreatedAt.Before(resourceCreationCutoff)
 	}
 
+	billingSinkAgeFilter := func(logSink *logging.LogSink) bool {
+		createdAt, err := time.Parse(time.RFC3339, logSink.CreateTime)
+		if err != nil {
+			logger.Printf("Failed to parse CreateTime for tagKey [%s], skipping it, error [%s]", logSink.ResourceName, err.Error())
+			return false
+		}
+		return createdAt.Before(resourceCreationCutoff)
+	}
+
 	projectDeleteRequestedFilter := func(projectID string) bool {
 		p, err := cloudResourceManagerService.Projects.Get(projectID).Context(ctx).Do()
 		if err != nil {
@@ -586,6 +567,24 @@ func invoke(ctx context.Context) {
 		}
 	}
 
+	removeBillingSinks := func(billing string) {
+		logger.Printf("Try to remove billing account log sinks from billing account [%s]", billing)
+		parent := fmt.Sprintf("billingAccounts/%s", billing)
+		sinkList, err := billingSinkService.List(parent).PageSize(billingSinksPageSize).Context(ctx).Do()
+		if err != nil {
+			logger.Printf("Failed to list billing account log sinks from billing account [%s], error [%s]", billing, err.Error())
+			return
+		}
+		for _, sink := range sinkList.Sinks {
+			if sink.Name != "_Required" && sink.Name != "_Default"  && billingSinkAgeFilter(sink) && checkIfNameIncluded(sink.ResourceName, targetBillingSinks) {
+				_, err = billingSinkService.Delete(sink.ResourceName).Context(ctx).Do()
+				if err != nil {
+					logger.Printf("Failed to delete billing account log sink [%s] from billing account [%s], error [%s]", sink.ResourceName, billing, err.Error())
+				}
+			}
+		}
+	}
+
 	removeFirewallPolicies := func(folder string) {
 		logger.Printf("Try to remove Firewall Policies from folder [%s]", folder)
 		firewallPolicyList, err := firewallPoliciesService.List().ParentId(folder).Context(ctx).Do()
@@ -743,6 +742,10 @@ func invoke(ctx context.Context) {
 	if cleanUpCaiFeeds {
 		removeFeedsByName(organizationId)
 	}
+
+	if cleanUpBillingSinks {
+		removeBillingSinks(billingAccount)
+	}
 }
 
 func CleanUpProjects(ctx context.Context, m PubSubMessage) error {