Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
d9aa4d1
copy spec changes
edgao Aug 23, 2023
4d10768
logistics
edgao Aug 23, 2023
5e0f08f
remove normalization from build
edgao Aug 23, 2023
e160ad6
remove unnecessary change
edgao Aug 23, 2023
dd0a8db
inject param to stagingcsvgenerator+stagingconsumerfactory
edgao Aug 23, 2023
0e1f518
inject to GcsUtils
edgao Aug 23, 2023
b5ae4d3
hardcode snowflake
edgao Aug 23, 2023
0c8248d
hardcode in bigquery
edgao Aug 23, 2023
1dbd194
Merge branch 'master' into edgao/dv2/release
edgao Aug 24, 2023
5a978ed
derp, fix default behavior
edgao Aug 24, 2023
050c1d9
derp
edgao Aug 24, 2023
46e1e67
derp
edgao Aug 24, 2023
0f14339
maybe make bigquery tests pass
edgao Aug 24, 2023
7c4cd0c
fix snowflake tests?
edgao Aug 24, 2023
2402243
Merge branch 'master' into edgao/dv2/release
edgao Aug 24, 2023
dc619e0
fix snowflake unit tests
edgao Aug 24, 2023
a7f92d1
more snowflake test fix
edgao Aug 24, 2023
86f4ffc
Automated Commit - Format and Process Resources Changes
edgao Aug 24, 2023
b244993
disable legacy DATs on snowflake + bigquery
edgao Aug 24, 2023
9c3bcac
Update upgrade copy
evantahler Aug 24, 2023
cb5e32b
Merge branch 'master' into edgao/dv2/release
evantahler Aug 24, 2023
be9b921
also disable these tests :shrug:
edgao Aug 25, 2023
155e391
one more
edgao Aug 25, 2023
05b58a3
prevent concurrent T+D
edgao Aug 25, 2023
c31f1ff
add better locks
edgao Aug 28, 2023
723a13f
Merge branch 'master' into edgao/dv2/release
evantahler Aug 28, 2023
bda59c6
Merge branch 'edgao/dv2/release' into edgao/dv2/snowflake/locks
edgao Aug 28, 2023
8ca2c27
Automated Commit - Formatting Changes
evantahler Aug 28, 2023
66648a7
git pls
edgao Aug 28, 2023
9ec91ad
add mustRun parameter for explicitnes
edgao Aug 28, 2023
3b2f818
more comments
edgao Aug 28, 2023
c117798
Automated Commit - Formatting Changes
edgao Aug 28, 2023
5954328
Merge branch 'edgao/dv2/release' into edgao/dv2/snowflake/locks
edgao Aug 28, 2023
8135cf0
derp. also make bigquery staging lock things
edgao Aug 29, 2023
9270ab6
use putIfAbsent
edgao Aug 29, 2023
d9cdcc4
Merge branch 'master' into edgao/dv2/snowflake/locks
edgao Aug 29, 2023
0aa4d2b
logistics
edgao Aug 29, 2023
7e94392
Merge branch 'master' into edgao/dv2/snowflake/locks
edgao Aug 30, 2023
b30707e
better putIfAbsent
edgao Aug 30, 2023
7d0f922
Merge branch 'master' into edgao/dv2/snowflake/locks
edgao Sep 6, 2023
e214fec
botched merge
edgao Sep 7, 2023
10b8356
refactor
edgao Sep 7, 2023
85fc426
prevent incremental T+D for bq staging
edgao Sep 7, 2023
a032c35
Merge branch 'master' into edgao/dv2/snowflake/locks
edgao Sep 12, 2023
cf229de
spotbugs
edgao Sep 12, 2023
0d23374
why no autoformat?
edgao Sep 12, 2023
3fb9769
Automated Commit - Format and Process Resources Changes
edgao Sep 12, 2023
c359ce6
Merge branch 'master' into edgao/dv2/snowflake/locks
edgao Sep 12, 2023
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
maybe make bigquery tests pass
  • Loading branch information
edgao committed Aug 24, 2023
commit 0f14339d38a2f9d55c79ce16bbc00e8ba56c168a
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
import io.airbyte.db.bigquery.BigQueryResultSet;
import io.airbyte.db.bigquery.BigQuerySourceOperations;
import io.airbyte.integrations.base.JavaBaseConstants;
import io.airbyte.integrations.base.destination.typing_deduping.StreamId;
import io.airbyte.integrations.destination.NamingConventionTransformer;
import io.airbyte.integrations.destination.StandardNameTransformer;
import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQuerySqlGenerator;
import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
import java.io.IOException;
Expand Down Expand Up @@ -99,11 +101,6 @@ protected boolean supportIncrementalSchemaChanges() {
return true;
}

@Override
protected boolean supportsInDestinationNormalization() {
return true;
}

@Override
protected Optional<NamingConventionTransformer> getNameTransformer() {
return Optional.of(NAME_TRANSFORMER);
Expand Down Expand Up @@ -131,21 +128,14 @@ protected String getDefaultSchema(final JsonNode config) {
return BigQueryUtils.getDatasetId(config);
}

@Override
protected List<JsonNode> retrieveNormalizedRecords(final TestDestinationEnv testEnv, final String streamName, final String namespace)
throws Exception {
final String tableName = namingResolver.getIdentifier(streamName);
final String schema = namingResolver.getIdentifier(namespace);
return retrieveRecordsFromTable(tableName, schema);
}

@Override
protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
final String streamName,
final String namespace,
final JsonNode streamSchema)
throws Exception {
return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namingResolver.getIdentifier(namespace))
final StreamId streamId = new BigQuerySqlGenerator(null).buildStreamId(namespace, streamName, JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE);
return retrieveRecordsFromTable(streamId.rawName(), streamId.rawNamespace())
.stream()
.map(node -> node.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())
.map(Jsons::deserialize)
Expand All @@ -159,7 +149,7 @@ protected List<JsonNode> retrieveRecordsFromTable(final String tableName, final
QueryJobConfiguration
.newBuilder(
String.format("SELECT * FROM `%s`.`%s` order by %s asc;", schema, tableName,
JavaBaseConstants.COLUMN_NAME_EMITTED_AT))
JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT))
.setUseLegacySql(false)
.setConnectionProperties(Collections.singletonList(ConnectionProperty.of("time_zone", "UTC")))
.build();
Expand Down