Skip to content

Commit 68ae711

Browse files
author
Marco Scalzo
committed
import spotless form our plugin
1 parent 3318740 commit 68ae711

File tree

4 files changed

+58
-80
lines changed

4 files changed

+58
-80
lines changed

‎client-spark/build.gradle.kts

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
plugins {
22
java
33
id("com.diffplug.spotless")
4+
id("whitefox.java-conventions")
45
}
56

67
group = "io.whitefox"
@@ -13,7 +14,7 @@ repositories {
1314
val hadoopVersion = "3.3.6"
1415
dependencies {
1516
// DELTA
16-
implementation(String.format("org.apache.hadoop:hadoop-common:%s", hadoopVersion))
17+
testImplementation(String.format("org.apache.hadoop:hadoop-common:%s", hadoopVersion))
1718
testImplementation("io.delta:delta-sharing-spark_2.12:1.0.2")
1819

1920
//SPARK
@@ -28,9 +29,7 @@ dependencies {
2829

2930
// region code formatting
3031
spotless {
31-
java {
32-
googleJavaFormat("1.14.0")
33-
}
32+
java {}
3433
}
3534
// endregion
3635

‎client-spark/src/test/java/client/ITDeltaSharingClient.java

Lines changed: 24 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,17 @@
1919

2020
public class ITDeltaSharingClient implements DatasetComparer {
2121

22-
private final String tablePath =
23-
String.format(
24-
"%s#%s.%s.%s",
25-
getClass().getClassLoader().getResource("MrFoxProfile.json"),
26-
"s3share",
27-
"s3schema",
28-
"s3Table1");
29-
30-
private final SparkSession spark =
31-
SparkSession.builder()
32-
.appName("delta sharing client test")
33-
.master("local[1, 4]")
34-
.getOrCreate();
22+
private final String tablePath = String.format(
23+
"%s#%s.%s.%s",
24+
getClass().getClassLoader().getResource("MrFoxProfile.json"),
25+
"s3share",
26+
"s3schema",
27+
"s3Table1");
28+
29+
private final SparkSession spark = SparkSession.builder()
30+
.appName("delta sharing client test")
31+
.master("local[1, 4]")
32+
.getOrCreate();
3533

3634
@BeforeAll
3735
static void initStorageManager() throws URISyntaxException, JsonProcessingException {
@@ -41,22 +39,19 @@ static void initStorageManager() throws URISyntaxException, JsonProcessingExcept
4139
@Test
4240
void showS3Table1withQueryTableApi() {
4341
var ds = spark.read().format("deltaSharing").load(tablePath);
44-
var expectedSchema =
45-
new StructType(
46-
new StructField[] {
47-
new StructField("id", DataType.fromDDL("long"), true, new Metadata(GenMap.empty()))
48-
});
49-
var expectedData =
50-
spark
51-
.createDataFrame(
52-
List.of(
53-
new MrFoxDeltaTableSchema(0),
54-
new MrFoxDeltaTableSchema(3),
55-
new MrFoxDeltaTableSchema(2),
56-
new MrFoxDeltaTableSchema(1),
57-
new MrFoxDeltaTableSchema(4)),
58-
MrFoxDeltaTableSchema.class)
59-
.toDF();
42+
var expectedSchema = new StructType(new StructField[] {
43+
new StructField("id", DataType.fromDDL("long"), true, new Metadata(GenMap.empty()))
44+
});
45+
var expectedData = spark
46+
.createDataFrame(
47+
List.of(
48+
new MrFoxDeltaTableSchema(0),
49+
new MrFoxDeltaTableSchema(3),
50+
new MrFoxDeltaTableSchema(2),
51+
new MrFoxDeltaTableSchema(1),
52+
new MrFoxDeltaTableSchema(4)),
53+
MrFoxDeltaTableSchema.class)
54+
.toDF();
6055

6156
assertEquals(expectedSchema.json(), ds.schema().json());
6257
assertEquals(5, ds.count());

‎client-spark/src/test/java/utils/EnvReader.java

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,9 @@ private Properties readProperties() {
2020
Properties properties = new Properties();
2121
FileInputStream input = null;
2222
try {
23-
input =
24-
new FileInputStream(
25-
String.format(
26-
"%s/.env",
27-
Paths.get(".").toAbsolutePath().getParent().getParent().toUri().getPath()));
23+
input = new FileInputStream(String.format(
24+
"%s/.env",
25+
Paths.get(".").toAbsolutePath().getParent().getParent().toUri().getPath()));
2826
properties.load(input);
2927
} catch (IOException e) {
3028
e.printStackTrace();

‎client-spark/src/test/java/utils/StorageManagerInitializer.java

Lines changed: 28 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -37,29 +37,24 @@ public void initStorageManager() throws JsonProcessingException, URISyntaxExcept
3737
createSchemaRequest(objectWriter),
3838
addTableToSchemaRequest(objectWriter))
3939
.stream()
40-
.forEach(
41-
request -> {
42-
try {
43-
callWhiteFoxServer(httpClient, request);
44-
} catch (Throwable e) {
45-
throw new RuntimeException(e);
46-
}
47-
});
40+
.forEach(request -> {
41+
try {
42+
callWhiteFoxServer(httpClient, request);
43+
} catch (Throwable e) {
44+
throw new RuntimeException(e);
45+
}
46+
});
4847
}
4948

5049
private HttpRequest addTableToSchemaRequest(ObjectWriter writer)
5150
throws JsonProcessingException, URISyntaxException {
52-
AddTableToSchemaInput addTableToSchemaInput =
53-
new AddTableToSchemaInput(
54-
"s3Table1", new AddTableToSchemaInput.TableReference("MrFoxProvider", "s3Table1"));
51+
AddTableToSchemaInput addTableToSchemaInput = new AddTableToSchemaInput(
52+
"s3Table1", new AddTableToSchemaInput.TableReference("MrFoxProvider", "s3Table1"));
5553
return HttpRequest.newBuilder()
5654
.header("content", "application/json")
5755
.POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(addTableToSchemaInput)))
58-
.uri(
59-
URI.create(
60-
String.format(
61-
"%s/%s/%s/%s/tables",
62-
server, "/whitefox-api/v1/shares", "s3share", "s3schema")))
56+
.uri(URI.create(String.format(
57+
"%s/%s/%s/%s/tables", server, "/whitefox-api/v1/shares", "s3share", "s3schema")))
6358
.build();
6459
}
6560

@@ -68,10 +63,8 @@ private HttpRequest createSchemaRequest(ObjectWriter writer)
6863
return HttpRequest.newBuilder()
6964
.POST(HttpRequest.BodyPublishers.noBody())
7065
.header("content", "application/json")
71-
.uri(
72-
URI.create(
73-
String.format(
74-
"%s/%s/%s/%s", server, "/whitefox-api/v1/shares", "s3share", "s3schema")))
66+
.uri(URI.create(
67+
String.format("%s/%s/%s/%s", server, "/whitefox-api/v1/shares", "s3share", "s3schema")))
7568
.build();
7669
}
7770

@@ -88,21 +81,18 @@ private HttpRequest createShareRequest(ObjectWriter writer)
8881

8982
private HttpRequest createTableRequest(ObjectWriter writer)
9083
throws JsonProcessingException, URISyntaxException {
91-
CreateTableInput createTableInput =
92-
new CreateTableInput(
93-
"s3Table1",
94-
"",
95-
true,
96-
new CreateTableInput.DeltaTableProperties(
97-
"delta", "s3a://whitefox-s3-test-bucket/delta/samples/delta-table", null));
84+
CreateTableInput createTableInput = new CreateTableInput(
85+
"s3Table1",
86+
"",
87+
true,
88+
new CreateTableInput.DeltaTableProperties(
89+
"delta", "s3a://whitefox-s3-test-bucket/delta/samples/delta-table", null));
9890

9991
return HttpRequest.newBuilder()
10092
.POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(createTableInput)))
10193
.header("content", "application/json")
102-
.uri(
103-
URI.create(
104-
String.format(
105-
"%s/%s/%s/tables", server, "whitefox-api/v1/providers", "MrFoxProvider")))
94+
.uri(URI.create(
95+
String.format("%s/%s/%s/tables", server, "whitefox-api/v1/providers", "MrFoxProvider")))
10696
.build();
10797
}
10898

@@ -117,17 +107,13 @@ private HttpRequest createProviderRequest(ObjectWriter writer) throws JsonProces
117107

118108
private HttpRequest createStorageRequest(ObjectWriter writer) throws JsonProcessingException {
119109
S3TestConfig s3TestConfig = envReader.readS3TestConfig();
120-
CreateStorage createStorage =
121-
new CreateStorage(
122-
"MrFoxStorage",
123-
"",
124-
"s3",
125-
new S3Properties(
126-
new S3Properties.AwsCredentials(
127-
s3TestConfig.getAccessKey(),
128-
s3TestConfig.getSecretKey(),
129-
s3TestConfig.getRegion())),
130-
true);
110+
CreateStorage createStorage = new CreateStorage(
111+
"MrFoxStorage",
112+
"",
113+
"s3",
114+
new S3Properties(new S3Properties.AwsCredentials(
115+
s3TestConfig.getAccessKey(), s3TestConfig.getSecretKey(), s3TestConfig.getRegion())),
116+
true);
131117
return HttpRequest.newBuilder()
132118
.POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(createStorage)))
133119
.header("content", "application/json")

0 commit comments

Comments
 (0)