Dataflow cost metrics

manifest.json

@@ -8,6 +8,11 @@
       "matches": ["https://console.cloud.google.com/*"],
       "js": ["./dist/custom-toolbar-color.js"]
 
+    },
+    {
+      "matches": ["https://console.cloud.google.com/dataflow/jobsDetail/*"],
+      "js": ["./dist/dataflow-cost-metrics.js"]
+
     }
   ],
 	"browser_action": {

package.json

@@ -11,6 +11,7 @@
     "clean": "run-p clean:*",
     "build:clean-dist": "rimraf dist",
     "build:cs-toolbar": "parcel build ./src/content-scripts/custom-toolbar-color.ts",
+    "build:cs-cost-metrics": "parcel build ./src/content-scripts/dataflow-cost-metrics.ts",
     "build:popup-html": "parcel build ./src/popup/popup.html --public-url ./",
     "build": "run-p build:*",
     "zip": "bestzip gcpimp.zip manifest.json dist src/assets",
@@ -28,6 +29,7 @@
   },
   "homepage": "https://github.com/Doctusoft/gcpimp#readme",
   "devDependencies": {
+    "@types/bytes": "^3.0.0",
     "@types/chrome": "0.0.69",
     "@types/react": "^16.4.6",
     "@types/react-dom": "^16.0.6",
@@ -40,6 +42,7 @@
     "typescript": "^2.9.2"
   },
   "dependencies": {
+    "bytes": "^3.0.0",
     "react": "^16.4.1",
     "react-dom": "^16.4.1",
     "react-md": "^1.4.1"

src/common/billing.service.ts

@@ -0,0 +1,109 @@
+const NANOS = 1e-9;
+
+interface GoogleService {
+	name: string;
+	serviceId: string;
+	displayName: string;
+}
+
+interface GoogleSKUPricingInfo {
+	summary: string;
+	pricingExpression: {
+		usageUnit: string;
+		usegeUnitDescription: string;
+		baseUnit: string;
+		baseUnitConversionFactor: number;
+		displayQuantity: number;
+		tieredRates: {
+			startUsageAmount: 0,
+			unitPrice: {
+				currencyCode: string;
+				units: string;
+				nanos: number;
+			},
+		}[];
+	};
+	currencyConversionRate: number;
+	effectiveTime: string;
+}
+
+interface CostMetricPrice {
+	amount: number;
+	currencyCode: string;
+}
+
+export interface DataflowCostMetricPrices {
+		vCPUTimeBatch: CostMetricPrice;
+		vCPUTimeStreaming: CostMetricPrice;
+		localDiskTimePdStandard: CostMetricPrice;
+		localDiskTimePDSSD: CostMetricPrice;
+		ramTime: CostMetricPrice;
+}
+
+interface GoogleSKU {
+	name: string;
+	skuId: string;
+	description: string;
+	category: {};
+	serviceRegions: string[];
+	pricingInfo: GoogleSKUPricingInfo[];
+	serviceProviderName: string;
+}
+
+const API_KEY = 'AIzaSyBZVfVwDKpduSuNOJlvWildIeQ5AsNtnWM';
+
+let services: GoogleService[];
+let dataflowPrices: GoogleSKU[];
+
+class BillingService {
+
+	private async listServices() {
+		const response = await fetch(`https://cloudbilling.googleapis.com/v1/services?key=${API_KEY}`);
+		const serviceList: { services: GoogleService[] } = await response.json();
+		services = serviceList.services;
+		return services;
+	}
+
+	private async getDataflowServiceId() {
+		const currentServices = services || await this.listServices();
+		return currentServices.find(svc => svc.displayName === 'Cloud Dataflow').serviceId;
+	}
+
+	private async listDataflowPrices() {
+		if (dataflowPrices) {
+			return dataflowPrices;
+		}
+		const serviceId = await this.getDataflowServiceId();
+		const response = await fetch(`https://cloudbilling.googleapis.com/v1/services/${serviceId}/skus?key=${API_KEY}`);
+		const body: { skus: GoogleSKU[] } = await response.json();
+		dataflowPrices = body.skus;
+		return dataflowPrices;
+	}
+
+	private getPrice(service: string, prices: GoogleSKU[]): CostMetricPrice {
+		const sku = prices.find(s => s.description.includes(service));
+		if (!sku) {
+			throw new Error('Couldn\'t find sku for service:' + service + ' among: ' + prices);
+		}
+		const unitPrice = sku.pricingInfo[0].pricingExpression.tieredRates[0].unitPrice;
+		return {
+			amount: unitPrice.nanos * NANOS,
+			currencyCode: unitPrice.currencyCode,
+		};
+	}
+
+	public async getDataflowCostMetricsPrices(region: string): Promise<DataflowCostMetricPrices> {
+		const prices = await this.listDataflowPrices();
+		const regionalPrices = prices.filter(sku => sku.serviceRegions.includes(region));
+		return {
+			vCPUTimeBatch: this.getPrice('vCPU Time Batch', regionalPrices),
+			vCPUTimeStreaming: this.getPrice('vCPU Time Streaming', regionalPrices),
+			localDiskTimePdStandard: this.getPrice('Local Disk Time PD Standard', regionalPrices),
+			localDiskTimePDSSD: this.getPrice('Local Disk Time PD SSD', regionalPrices),
+			ramTime: this.getPrice('RAM Time', regionalPrices),
+		};
+	}
+
+}
+
+export const billingService = Object.freeze(new BillingService());

src/common/utils.ts

@@ -1,2 +1,17 @@
 
 export const getProjectId = (url: string) => new URL(url).searchParams.get('project');
+
+type UrlChangeHandler = (url: string) => any;
+export const onUrlChange = (handler: UrlChangeHandler) => {
+	let oldHref = document.location.href;
+	const body = document.querySelector('body');
+	const observer = new MutationObserver(() => {
+		const newHref = document.location.href;
+		if (oldHref !== newHref) {
+			oldHref = newHref;
+			handler(newHref);
+		}
+	});
+	observer.observe(body, { childList: true, attributes: true, subtree: true });
+	return () => observer.disconnect();
+};

src/content-scripts/custom-toolbar-color.ts

@@ -1,6 +1,6 @@
 import { configService } from '../common/config.service';
 import { MessageType } from '../common/message';
-import { getProjectId } from '../common/utils';
+import { getProjectId, onUrlChange } from '../common/utils';
 
 const TOOLBAR_SELECTOR = '.pcc-platform-bar-container';
 
@@ -25,13 +25,4 @@ chrome.runtime.onMessage.addListener(message => {
 	}
 });
 
-let oldHref = document.location.href;
-const body = document.querySelector('body');
-const observer = new MutationObserver(() => {
-	if (oldHref !== document.location.href) {
-		oldHref = document.location.href;
-		customizeToolbarColor();
-	}
-});
-
-observer.observe(body, { childList: true, attributes: true, subtree: true });
+onUrlChange(() => customizeToolbarColor());

src/content-scripts/dataflow-cost-metrics.ts

@@ -0,0 +1,174 @@
+import { format, parse } from 'bytes';
+import { billingService, DataflowCostMetricPrices } from '../common/billing.service';
+
+const daxServiceMetrics = 'dax-service-metrics';
+const list = '.p6n-kv-list';
+const listItemClassName = 'p6n-kv-list-item';
+const gcpCostMetric = 'gcpimp-cost-metric';
+
+const getDaxServiceMetrics = () => document.querySelector(daxServiceMetrics) as HTMLElement;
+const getList = () => getDaxServiceMetrics().querySelector(list);
+const getListItems = () => Array.from(getList().querySelectorAll('.' + listItemClassName));
+const getCostMetrics = () => Array.from(getList().querySelectorAll('.' + gcpCostMetric));
+
+const createListItem = (key: string, value: string) => {
+	const newListItem = document.createElement('div');
+	newListItem.className = listItemClassName + ' ' + gcpCostMetric;
+	newListItem.innerHTML = `
+	<div class="p6n-kv-list-key">
+		<span>${key}</span>
+	</div>
+	<div class="p6n-kv-list-values">
+		<div class="p6n-kv-list-value">
+			<span>${value}</span>
+		</div>
+	</div>
+	`;
+	return newListItem;
+};
+
+async function elementLoaded<T>(supplier: () => T, parentElement = document.body) {
+	const loaded = supplier();
+	if (!!loaded) {
+		return loaded;
+	}
+	const watchMutations = new Promise<T>(resolve => {
+		const observer = new MutationObserver(() => {
+			const result = supplier();
+			if (!!result) {
+				observer.disconnect();
+				resolve(result);
+			}
+		});
+		observer.observe(parentElement, { childList: true, subtree: true });
+	});
+	await watchMutations;
+}
+
+const selectorLoaded = (selector: string, parentElement: HTMLElement) =>
+	elementLoaded(() => parentElement.querySelector(selector), parentElement);
+
+type JobProperties = Record<string, string> & {
+	zone?: string;
+	region?: string;
+	Region?: string;
+	'Job type'?: 'Streaming' | 'Batch';
+	'Current PD'?: string;
+	'Current SSD PD'?: string;
+	'Current memory'?: string;
+	'Current vCPUs'?: string;
+	'Total PD time'?: string;
+	'Total SSD PD time'?: string;
+	'Total memory time'?: string;
+	'Total vCPU time'?: string;
+};
+
+function getJobProperties(): JobProperties {
+	const listItems = Array.from(document.querySelectorAll('.' + listItemClassName));
+	const itemValueMap: JobProperties = {};
+	listItems.forEach(item => {
+		const keyEl = item.querySelector('.p6n-kv-list-key>div, .p6n-kv-list-key>span');
+		if (!keyEl) {
+			return;
+		}
+		const key = keyEl.textContent.trim();
+		const value = item.querySelector('.p6n-kv-list-value').textContent.trim();
+		itemValueMap[key] = value;
+	});
+	return itemValueMap;
+}
+
+const zoneToRegion = (zone: string) => {
+	if (!zone) {
+		return;
+	}
+	const zoneParts = zone.split('-');
+	zoneParts.splice(-1, 1);
+	return zoneParts.join('-');
+};
+
+function findRegion(jobProperties = getJobProperties()) {
+	const zoneRegion = jobProperties.zone ? zoneToRegion(jobProperties.zone) : undefined;
+	const overriddenRegion = jobProperties.region;
+	const defaultRegion = jobProperties.Region;
+
+	return zoneRegion || overriddenRegion || defaultRegion;
+}
+
+const gbValue = (value: string) => {
+	const valueStripped = (value || '').replace('hr', '').trim();
+	const parseGBValueToBytes = parse(valueStripped);
+	const formatBytesToGBString = format(parseGBValueToBytes, { unit: 'GB', unitSeparator: ' ' });
+	const stringGbValue = (formatBytesToGBString || '').split(' ')[0];
+	return Number.parseFloat(stringGbValue);
+};
+
+const getCpuPrice = (prices: DataflowCostMetricPrices, job: JobProperties) =>
+	job['Job type'] === 'Batch' ? prices.vCPUTimeBatch : prices.vCPUTimeStreaming;
+
+function calculateCurrentCost(prices: DataflowCostMetricPrices, job = getJobProperties()) {
+	const currentPD = gbValue(job['Current PD'] || '');
+	const currentPDSSD = gbValue(job['Current SSD PD'] || '');
+	const currentMemory = gbValue(job['Current memory'] || '');
+	const currentVCPUs = Number.parseFloat(job['Current vCPUs'] || '');
+
+	const cpuPrice = getCpuPrice(prices, job);
+
+	const cost =
+		currentPD * prices.localDiskTimePdStandard.amount +
+		currentPDSSD * prices.localDiskTimePDSSD.amount +
+		currentMemory * prices.ramTime.amount +
+		currentVCPUs * cpuPrice.amount;
+
+	return cost.toFixed(2) + ' ' + prices.ramTime.currencyCode + '/ hr';
+}
+
+function calculateTotalCost(prices: DataflowCostMetricPrices, job = getJobProperties()) {
+	const totalPDTime = gbValue(job['Total PD time'] || '');
+	const totalPDSSDTime = gbValue(job['Total SSD PD time'] || '');
+	const totalMemoryTime = gbValue(job['Total memory time'] || '');
+	const totalVCPUTime = Number.parseFloat((job['Total vCPU time'] || '').split(' ')[0]);
+
+	const cpuPrice = getCpuPrice(prices, job);
+
+	const cost =
+		totalPDTime * prices.localDiskTimePdStandard.amount +
+		totalPDSSDTime * prices.localDiskTimePDSSD.amount +
+		totalMemoryTime * prices.ramTime.amount +
+		totalVCPUTime * cpuPrice.amount;
+	return cost.toFixed(2) + ' ' + prices.ramTime.currencyCode;
+}
+
+async function appendCostMetrics() {
+	await elementLoaded(getDaxServiceMetrics);
+	await elementLoaded(getList, getDaxServiceMetrics());
+	const job = getJobProperties();
+	const region = findRegion(job);
+	if (!region) {
+		return;
+	}
+	const prices = await billingService.getDataflowCostMetricsPrices(region);
+
+	const listEl = getList();
+
+	const currentCost = calculateCurrentCost(prices, job);
+	const totalCost = calculateTotalCost(prices, job);
+	listEl.appendChild(createListItem('Current cost', currentCost));
+	listEl.appendChild(createListItem('Total cost', totalCost));
+}
+
+const removeAddedCostMetrics = () =>
+	Array.from(document.querySelectorAll('.' + gcpCostMetric)).forEach(el => el.remove());
+
+async function listenToMetricsChanges() {
+	await elementLoaded(getDaxServiceMetrics);
+	await elementLoaded(getList, getDaxServiceMetrics());
+
+	return setInterval(() => {
+		removeAddedCostMetrics();
+		appendCostMetrics();
+	}, 1000);
+}
+
+appendCostMetrics();
+listenToMetricsChanges();

tsconfig.json

@@ -4,7 +4,8 @@
         "target": "es5",
         "lib": [
           "es2015",
-          "dom"
+          "dom",
+          "es2016.array.include"
         ],
         "noImplicitAny": false,
         "strictNullChecks": false,