From d48777b02e1fe29d8da2d1b596cc439e75b96f02 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Wed, 23 Oct 2024 09:56:18 -0700 Subject: [PATCH] More improvements to ensured traced and untraced are fairly used. --- observability-benchmarking/README.md | 3 - observability-benchmarking/benchmark.js | 298 ------------------------ observability-benchmarking/package.json | 29 --- observability-test/comparisons.ts | 99 +++++--- package.json | 1 + 5 files changed, 65 insertions(+), 365 deletions(-) delete mode 100644 observability-benchmarking/README.md delete mode 100644 observability-benchmarking/benchmark.js delete mode 100644 observability-benchmarking/package.json diff --git a/observability-benchmarking/README.md b/observability-benchmarking/README.md deleted file mode 100644 index 1207b2fb8..000000000 --- a/observability-benchmarking/README.md +++ /dev/null @@ -1,3 +0,0 @@ -## Observability Benchmarking - -This code serves to produce an impact assessment of the effect of adding observability to each call. diff --git a/observability-benchmarking/benchmark.js b/observability-benchmarking/benchmark.js deleted file mode 100644 index 33cbe1e0d..000000000 --- a/observability-benchmarking/benchmark.js +++ /dev/null @@ -1,298 +0,0 @@ -/*! - * Copyright 2024 Google LLC. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -const {MutationSet, Spanner} = require('@google-cloud/spanner'); -const {readFileSync} = require('fs'); - -async function runComparisons(v1JSONFile, v2JSONFile) { - const v1JSON = JSON.parse(readFileSync(v1JSONFile)); - const v2JSON = JSON.parse(readFileSync(v2JSONFile)); - return compareDifferences(v1JSON, v2JSON); -} - -function main() { - if (process.argv[2] === 'compare') { - runComparisons(...process.argv.slice(3)); - } else { - runBenchmarking(...process.argv.slice(2)); - } -} - -async function runBenchmarking(projectId, instanceId, databaseId) { - // Otherwise run the benchmarks. - const spanner = new Spanner({ - projectId: projectId, - observabilityOptions: { - enableExtendedTracing: true, - }, - }); - - const instance = spanner.instance(instanceId); - const database = instance.database(databaseId); - - const runners = [ - databaseWriteAtLeastOnce, - dqlSelect1, - dqlSelectWithSyntaxError, - dmlDeleteThenInsert, - dmlWithDuplicates, - databasBatcheCreateSessions, - databasGetSessions, - databaseRun, - databaseRunWithSyntaxError, - databaseRunWithDelete, - databaseRunWithDeleteFromNonExistentTable, - ]; - - const nRuns = 1000; - const benchmarkValues = {}; - - let k = 0; - for (k=0; k < runners.length; k++){ - const fn = runners[k]; - const method = fn.name; - console.log(`Running ${k+1}/${runners.length} ${method}`); - const latencyL = []; - const ramL = []; - let i = 0; - - for (i = 0; i < nRuns; i++) { - const startTime = process.hrtime.bigint(); - const startHeapUsedBytes = process.memoryUsage().heapUsed; - try { - await fn(database); - } catch (e) { - } finally { - latencyL.push(process.hrtime.bigint() - startTime); - ramL.push(process.memoryUsage().heapUsed - startHeapUsedBytes); - } - } - - const lessComparator = (a, b) => { - if (a < b) return -1; - if (a > b) return 1; - return 0; - }; - latencyL.sort(lessComparator); - ramL.sort(lessComparator); - - benchmarkValues[method] = { - ram: percentiles(method, ramL, 'bytes'), - latency: percentiles(method, latencyL, 'time'), - }; - } - - BigInt.prototype.toJSON = function () { - return Number(this); - }; - console.log(JSON.stringify(benchmarkValues)); -} - -function percentiles(method, sortedValues, kind) { - const n = sortedValues.length; - const p50 = sortedValues[Math.floor(n * 0.5)]; - const p75 = sortedValues[Math.floor(n * 0.75)]; - const p90 = sortedValues[Math.floor(n * 0.90)]; - const p95 = sortedValues[Math.floor(n * 0.95)]; - const p99 = sortedValues[Math.floor(n * 0.99)]; - // console.log( - // `\tp50: ${p50}\n\tp75: ${p75}\n\tp95: ${p95}\n\tp99: ${p99}\n` - // ); - return { - p50: p50, p75: p75, p90: p90, p95: p95, p99:p99, - p50_s: humanize(p50, kind), - p75_s: humanize(p75, kind), - p90_s: humanize(p90, kind), - p95_s: humanize(p95, kind), - p99_s: humanize(p99, kind), - }; -} - -function humanize(values, kind) { - let converterFn = humanizeTime; - if (kind === 'bytes') { - converterFn = humanizeBytes; - } - return converterFn(values); -} - -const secondUnits = ['ns', 'us', 'ms', 's']; -const pastSecondUnits = [['min', 60], ['hr', 60], ['day', 24], ['week', 7], ['month', 30]]; -function humanizeTime(ns) { - let value = ns; - for (const unit of secondUnits) { - if (value < 1000) { - return `${value} ${unit}`; - } - value /= 1000n; - } - - for (const unitPlusValue of pastSecondUnits) { - const [unitName, divisor] = unitPlusValue; - if (value < divisor) { - return `${value} ${unitName}`; - } - value = value/divisor; - } - return `${value} ${units[units.length-1][0]}`; -} - -const bytesUnits = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'ExB']; -function humanizeBytes(b) { - let value = b; - for (const unit of bytesUnits) { - if (value < 1024) { - return `${value.toFixed(3)} ${unit}`; - } - value = value/1024; - } - - return `${value.toFixed(3)} ${bytesUnits[bytesUnits.length-1]}`; -} - -async function dqlSelect1(database) { - const [snapshot] = await database.getSnapshot(); - const [rows] = await snapshot.run('SELECT 1'); - await snapshot.end(); -} - -async function dqlSelectWithSyntaxError(database) { - const [snapshot] = await database.getSnapshot(); - try { - const [rows] = await snapshot.run('SELECT 1'); - } finally { - await snapshot.end(); - } -} - -async function dmlDeleteThenInsert(database) { - await database.runTransactionAsync(async tx => { - const [updateCount1] = await tx.runUpdate('DELETE FROM Singers WHERE 1=1'); - const [updateCount2] = await tx.runUpdate( - "INSERT INTO Singers(SingerId, firstName) VALUES(1, 'DTB')" - ); - await tx.commit(); - }); -} - -async function dmlWithDuplicates(database) { - return await database.runTransactionAsync(async tx => { - try { - const [updateCount1] = await tx.runUpdate( - "INSERT INTO Singers(SingerId, firstName) VALUES(1, 'DTB')" - ); - const [updateCount2] = await tx.runUpdate( - "INSERT INTO Singers(SingerId, firstName) VALUES(1, 'DTB')" - ); - } catch(e) { - } finally { - await tx.end(); - } - }); -} - -async function databasBatcheCreateSessions(database) { - return await database.batchCreateSessions(10); -} - -async function databasGetSessions(database) { - return await database.getSessions(); -} - -async function databaseRun(database) { - return await database.run('SELECT 1'); -} - -async function databaseRunWithSyntaxError(database) { - return await database.run('SELECT 10 p'); -} - -async function databaseRunWithDelete(database) { - return await database.run('DELETE FROM Singers WHERE 1=1'); -} - -async function databaseRunWithDeleteFromNonExistentTable(database) { - return await database.run('DELETE FROM NonExistent WHERE 1=1'); -} - -async function databaseWriteAtLeastOnce(database) { - const mutations = new MutationSet(); - mutations.upsert('Singers', { - SingerId: 1, - FirstName: 'Scarlet', - LastName: 'Terry', - }); - mutations.upsert('Singers', { - SingerId: 2, - FirstName: 'Marc', - LastName: 'Richards', - }); - - const [response, err] = await database.writeAtLeastOnce(mutations, {}); -} - -function compareDifferences(v1, v2) { - const percents = []; - for (const key in v1) { - const defV1 = v1[key]; - const ramV1 = defV1.ram; - const latencyV1 = defV1.latency; - const defV2 = v2[key]; - const ramV2 = defV2.ram; - const latencyV2 = defV2.latency; - - percents.push({ - key: key, - ramP50:calculatePercent(ramV1.p50, ramV2.p50).toFixed(2), - ramP75:calculatePercent(ramV1.p75, ramV2.p75).toFixed(2), - ramP90:calculatePercent(ramV1.p90, ramV2.p90).toFixed(2), - ramP95:calculatePercent(ramV1.p95, ramV2.p95).toFixed(2), - ramP99:calculatePercent(ramV1.p95, ramV2.p99).toFixed(2), - latP50: calculatePercent(latencyV1.p50, latencyV2.p50).toFixed(2), - latP75: calculatePercent(latencyV1.p75, latencyV2.p75).toFixed(2), - latP90: calculatePercent(latencyV1.p90, latencyV2.p90).toFixed(2), - latP95: calculatePercent(latencyV1.p95, latencyV2.p95).toFixed(2), - latP99: calculatePercent(latencyV1.p95, latencyV2.p99).toFixed(2), - }); - } - - // Deterministically print out results sorted by name. - percents.sort((a, b) => { - if (a.key < b.key) return -1; - if (a.key > b.key) return +1; - return 0; - }); - - for (const value of percents) { - console.log(`${value.key}`); - console.log(`\t p50 (%) p75 (%) p90 (%) p95 (%) p99 (%)`); - console.log(`\tRAM: ${value.ramP50} ${value.ramP75} ${value.ramP90} ${value.ramP95} ${value.ramP99}`); - console.log(`\tLat: ${value.latP50} ${value.latP75} ${value.latP90} ${value.latP95} ${value.latP99}`); - console.log(''); - } -} - -function calculatePercent(orig, updated) { - return (updated-orig)*100/orig; -} - -process.on('unhandledRejection', err => { - console.error(err.message); - process.exitCode = 1; -}); -main(); diff --git a/observability-benchmarking/package.json b/observability-benchmarking/package.json deleted file mode 100644 index 4a591cb19..000000000 --- a/observability-benchmarking/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "nodejs-observability-spanner", - "private": true, - "license": "Apache-2.0", - "author": "Google Inc.", - "repository": "googleapis/nodejs-spanner", - "files": [ - "*.js" - ], - "engines": { - "node": ">=14.0.0" - }, - "scripts": { - "test-with-archived": "mocha system-test --timeout 1600000", - "test": "mocha system-test/spanner.test.js --timeout 1600000" - }, - "dependencies": { - "@google-cloud/kms": "^4.0.0", - "@google-cloud/precise-date": "^4.0.0", - "@google-cloud/spanner": "^7.13.0", - "yargs": "^17.0.0", - "protobufjs": "^7.0.0" - }, - "devDependencies": { - "chai": "^4.2.0", - "mocha": "^9.0.0", - "p-limit": "^3.0.1" - } -} diff --git a/observability-test/comparisons.ts b/observability-test/comparisons.ts index 54b9ab709..3e3170b97 100644 --- a/observability-test/comparisons.ts +++ b/observability-test/comparisons.ts @@ -142,52 +142,81 @@ describe('Benchmarking', () => { server.tryShutdown(() => {}); }); + interface Tx { + begin(): Promise; + runUpdate(string): Promise; + end(): void; + } + const runners: Function[] = [ - async function databaseRunSelect1() { - return await database.run('SELECT 1'); + async function databaseRunSelect1Promise() { + if (traced) { + const [rows] = await (database as Database).run('SELECT 1'); + for (const row of rows) { + var _ = row; + } + } else { + const [rows] = await (database as DatabaseUntraced).run('SELECT 1'); + for (const row of rows) { + var _ = row; + } + } }, - /* - // Queries with bad syntax take 5+ seconds to run - // hence skew benchmark results - // https://github.com/googleapis/nodejs-spanner/issues/2173 - async function databaseRunSelect1BadSyntax() { - try { - return await database.run('SELECT 1p'); - } catch (e) { - return null; - } finally { - console.log('1B done'); - } - }, - */ + async function databaseRunSelect1Callback() { + return new Promise(resolve => { + const withErrRows = (err, rows) => { + for (const row of rows) { + const _ = row; + } + resolve(rows.length); + }; - async function databaseGetTransactionAsyncPlusTxnRunAwait() { - const promise = await database.getTransaction({ - optimisticLock: true, - requestOptions: {transactionTag: 'transaction-tag'}, + if (traced) { + (database as Database).run('SELECT 1', withErrRows); + } else { + (database as DatabaseUntraced).run('SELECT 1', withErrRows); + } }); - const transaction = promise[0]; - return await transaction.run('SELECT 1'); }, - async function databaseRunTransactionAsync() { - try { - let fn: Function; + /* + // TODO: Enable this code once mockspanner works well with getTransaction. + async function databaseGetTransactionAsync() { + return new Promise(resolve => { + const withErrTx = async(err, tx) => { + await tx!.begin(); + const [rows] = await tx!.run('SELECT 1'); + for (const row of rows) { + const _ = row; + } + await tx!.end(); + resolve(10); + }; + if (traced) { - fn = (database as Database).runTransactionAsync; + (database as Database).getTransction(withErrTx); } else { - fn = (database as DatabaseUntraced).runTransactionAsync; + (database as DatabaseUntraced).getTransction(withErrTx); } + }); + }, + */ - return await fn(async tx => { - await tx!.begin(); - const result = await tx!.runUpdate(updateSql); - tx!.end(); - return result; - }); - } catch (e) { - return null; + async function databaseRunTransactionAsync() { + const withTx = async (tx: Tx) => { + await tx!.begin(); + const result = await tx!.runUpdate(updateSql); + tx!.end(); + return result; + }; + + if (traced) { + return await (database as Database).runTransactionAsync(withTx); + } else { + return await (database as DatabaseUntraced).runTransactionAsync( + withTx + ); } }, ]; diff --git a/package.json b/package.json index 27ed2f743..e60bba0de 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "@google-cloud/precise-date": "^4.0.0", "@google-cloud/projectify": "^4.0.0", "@google-cloud/promisify": "^4.0.0", + "@google-cloud/spanner": "^7.14.0", "@grpc/proto-loader": "^0.7.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/context-async-hooks": "^1.26.0",