fix(spanner): do not export metrics during shutdown if prev export was less than 30 seconds ago (#12266)

* fix(spanner): do not export metrics during shutdown if prev export was less than 30 seconds ago

* Addressed the comments

* Addressed the comments

* fix go vet

---------

Co-authored-by: rahul yadav <[email protected]>
Co-authored-by: rahul2393 <[email protected]>

Author: sakthivelmanii

spanner/metrics.go

@@ -283,7 +283,7 @@ func newBuiltinMetricsTracerFactory(ctx context.Context, dbpath, compression str
 	var meterProvider *sdkmetric.MeterProvider
 	if metricsProvider == nil {
 		// Create default meter provider
-		mpOptions, err := builtInMeterProviderOptions(project, compression, tracerFactory.clientAttributes, opts...)
+		mpOptions, exporter, err := builtInMeterProviderOptions(project, compression, tracerFactory.clientAttributes, opts...)
 		if err != nil {
 			return tracerFactory, err
 		}
@@ -308,7 +308,7 @@ func newBuiltinMetricsTracerFactory(ctx context.Context, dbpath, compression str
 		}
 		tracerFactory.enabled = true
 		tracerFactory.shutdown = func(ctx context.Context) {
-			meterProvider.ForceFlush(ctx)
+			exporter.stop()
 			meterProvider.Shutdown(ctx)
 		}
 	} else {
@@ -326,11 +326,11 @@ func newBuiltinMetricsTracerFactory(ctx context.Context, dbpath, compression str
 	return tracerFactory, err
 }
 
-func builtInMeterProviderOptions(project, compression string, clientAttributes []attribute.KeyValue, opts ...option.ClientOption) ([]sdkmetric.Option, error) {
+func builtInMeterProviderOptions(project, compression string, clientAttributes []attribute.KeyValue, opts ...option.ClientOption) ([]sdkmetric.Option, *monitoringExporter, error) {
 	allOpts := createExporterOptions(opts...)
 	defaultExporter, err := newMonitoringExporter(context.Background(), project, compression, clientAttributes, allOpts...)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	var views []sdkmetric.View
 	for _, m := range grpcMetricsToEnable {
@@ -354,7 +354,7 @@ func builtInMeterProviderOptions(project, compression string, clientAttributes [
 			defaultExporter,
 			sdkmetric.WithInterval(defaultSamplePeriod),
 		),
-	), sdkmetric.WithView(views...)}, nil
+	), sdkmetric.WithView(views...)}, defaultExporter, nil
 }
 
 func (tf *builtinMetricsTracerFactory) createInstruments(meter metric.Meter) error {

spanner/metrics_monitoring_exporter.go

@@ -96,6 +96,10 @@ type monitoringExporter struct {
 	shutdown         chan struct{}
 	client           *monitoring.MetricClient
 	shutdownOnce     sync.Once
+
+	mu             sync.Mutex
+	stopExport     bool
+	lastExportedAt time.Time
 }
 
 func newMonitoringExporter(ctx context.Context, project, compression string, clientAttributes []attribute.KeyValue, opts ...option.ClientOption) (*monitoringExporter, error) {
@@ -107,11 +111,21 @@ func newMonitoringExporter(ctx context.Context, project, compression string, cli
 		projectID:        project,
 		compression:      compression,
 		clientAttributes: clientAttributes,
+		lastExportedAt:   time.Now().Add(-time.Minute),
 		client:           client,
 		shutdown:         make(chan struct{}),
 	}, nil
 }
 
+func (me *monitoringExporter) stop() {
+	// stop the exporter if last export happens within half-time of default sample period
+	me.mu.Lock()
+	defer me.mu.Unlock()
+	if time.Since(me.lastExportedAt) <= (defaultSamplePeriod / 2) {
+		me.stopExport = true
+	}
+}
+
 // ForceFlush does nothing, the exporter holds no state.
 func (e *monitoringExporter) ForceFlush(ctx context.Context) error { return ctx.Err() }
 
@@ -133,6 +147,12 @@ func (me *monitoringExporter) Export(ctx context.Context, rm *otelmetricdata.Res
 	default:
 	}
 
+	me.mu.Lock()
+	if me.stopExport {
+		me.mu.Unlock()
+		return nil
+	}
+	me.mu.Unlock()
 	return me.exportTimeSeries(ctx, rm)
 }
 
@@ -178,6 +198,9 @@ func (me *monitoringExporter) exportTimeSeries(ctx context.Context, rm *otelmetr
 		errs = append(errs, err)
 	}
 
+	me.mu.Lock()
+	me.lastExportedAt = time.Now()
+	me.mu.Unlock()
 	return errors.Join(errs...)
 }