From 27f7fb5af6cc7649e923c7b823712c31aeb010e9 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 6 Jun 2024 01:29:53 +0530 Subject: [PATCH 01/31] feat: integrate iceberg support via jvm grpc - add quarkus with grpc for jvm - expose required routes for appends, changes and setup tables --- .sdkmanrc | 4 + flow-jvm/.dockerignore | 5 + flow-jvm/.gitattributes | 3 + flow-jvm/.gitignore | 39 +++ flow-jvm/README.md | 56 ++++ flow-jvm/build.gradle | 69 ++++ flow-jvm/gradle.properties | 7 + flow-jvm/gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 58910 bytes .../gradle/wrapper/gradle-wrapper.properties | 5 + flow-jvm/gradlew | 185 +++++++++++ flow-jvm/gradlew.bat | 104 ++++++ flow-jvm/settings.gradle | 11 + flow-jvm/src/main/docker/Dockerfile.jvm | 97 ++++++ .../src/main/docker/Dockerfile.legacy-jar | 93 ++++++ flow-jvm/src/main/docker/Dockerfile.native | 27 ++ .../src/main/docker/Dockerfile.native-micro | 30 ++ .../main/java/io/peerdb/GreetingResource.java | 16 + .../jvm/DefaultExceptionHandlerProvider.java | 42 +++ .../io/peerdb/flow/jvm/ErrorInterceptor.java | 37 +++ .../java/io/peerdb/flow/jvm/GrpcError.java | 11 + .../iceberg/avro/AvroIcebergConverter.java | 61 ++++ .../jvm/iceberg/catalog/CatalogLoader.java | 49 +++ .../catalog/io/mapper/FileIOConfigMapper.java | 28 ++ .../catalog/io/mapper/S3IOConfigMapper.java | 30 ++ .../catalog/mapper/CatalogConfigMapper.java | 40 +++ .../catalog/mapper/HiveConfigMapper.java | 21 ++ .../catalog/mapper/JdbcCatalogMapper.java | 30 ++ .../jvm/iceberg/resource/IcebergResource.java | 89 ++++++ .../jvm/iceberg/service/IcebergService.java | 199 ++++++++++++ .../flow/jvm/iceberg/writer/TaskWriter.java | 4 + .../src/main/resources/application.properties | 1 + flow-jvm/src/main/resources/application.yaml | 24 ++ .../java/io/peerdb/GreetingResourceIT.java | 8 + .../java/io/peerdb/GreetingResourceTest.java | 20 ++ .../java/io/peerdb/IcebergServiceTest.java | 22 ++ flow/activities/flowable.go | 7 + flow/connectors/core.go | 6 + flow/connectors/iceberg/iceberg.go | 302 ++++++++++++++++++ flow/connectors/iceberg/qrep.go | 149 +++++++++ flow/connectors/utils/peers.go | 11 +- flow/logger/handler.go | 3 + protos/flow-jvm.proto | 106 ++++++ protos/peers.proto | 64 ++++ 43 files changed, 2113 insertions(+), 2 deletions(-) create mode 100644 .sdkmanrc create mode 100644 flow-jvm/.dockerignore create mode 100644 flow-jvm/.gitattributes create mode 100644 flow-jvm/.gitignore create mode 100644 flow-jvm/README.md create mode 100644 flow-jvm/build.gradle create mode 100644 flow-jvm/gradle.properties create mode 100644 flow-jvm/gradle/wrapper/gradle-wrapper.jar create mode 100644 flow-jvm/gradle/wrapper/gradle-wrapper.properties create mode 100755 flow-jvm/gradlew create mode 100644 flow-jvm/gradlew.bat create mode 100644 flow-jvm/settings.gradle create mode 100644 flow-jvm/src/main/docker/Dockerfile.jvm create mode 100644 flow-jvm/src/main/docker/Dockerfile.legacy-jar create mode 100644 flow-jvm/src/main/docker/Dockerfile.native create mode 100644 flow-jvm/src/main/docker/Dockerfile.native-micro create mode 100644 flow-jvm/src/main/java/io/peerdb/GreetingResource.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/ErrorInterceptor.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/GrpcError.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FileIOConfigMapper.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/JdbcCatalogMapper.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/TaskWriter.java create mode 100644 flow-jvm/src/main/resources/application.properties create mode 100644 flow-jvm/src/main/resources/application.yaml create mode 100644 flow-jvm/src/native-test/java/io/peerdb/GreetingResourceIT.java create mode 100644 flow-jvm/src/test/java/io/peerdb/GreetingResourceTest.java create mode 100644 flow-jvm/src/test/java/io/peerdb/IcebergServiceTest.java create mode 100644 flow/connectors/iceberg/iceberg.go create mode 100644 flow/connectors/iceberg/qrep.go create mode 100644 protos/flow-jvm.proto diff --git a/.sdkmanrc b/.sdkmanrc new file mode 100644 index 0000000000..8d3433b204 --- /dev/null +++ b/.sdkmanrc @@ -0,0 +1,4 @@ +# Enable auto-env through the sdkman_auto_env config +# Add key=value pairs of SDKs to use below +java=21.0.3-tem +quarkus=3.10.2 diff --git a/flow-jvm/.dockerignore b/flow-jvm/.dockerignore new file mode 100644 index 0000000000..4361d2fb38 --- /dev/null +++ b/flow-jvm/.dockerignore @@ -0,0 +1,5 @@ +* +!build/*-runner +!build/*-runner.jar +!build/lib/* +!build/quarkus-app/* \ No newline at end of file diff --git a/flow-jvm/.gitattributes b/flow-jvm/.gitattributes new file mode 100644 index 0000000000..4951bacfbd --- /dev/null +++ b/flow-jvm/.gitattributes @@ -0,0 +1,3 @@ +gradlew linguist-generated=true +gradlew.bat linguist-generated=true + diff --git a/flow-jvm/.gitignore b/flow-jvm/.gitignore new file mode 100644 index 0000000000..216783d79c --- /dev/null +++ b/flow-jvm/.gitignore @@ -0,0 +1,39 @@ +# Gradle +.gradle/ +build/ + +# Eclipse +.project +.classpath +.settings/ +bin/ + +# IntelliJ +.idea +*.ipr +*.iml +*.iws + +# NetBeans +nb-configuration.xml + +# Visual Studio Code +.vscode +.factorypath + +# OSX +.DS_Store + +# Vim +*.swp +*.swo + +# patch +*.orig +*.rej + +# Local environment +.env + +# Plugin directory +/.quarkus/cli/plugins/ diff --git a/flow-jvm/README.md b/flow-jvm/README.md new file mode 100644 index 0000000000..eda52dd049 --- /dev/null +++ b/flow-jvm/README.md @@ -0,0 +1,56 @@ +# flow-jvm + +This project uses Quarkus, the Supersonic Subatomic Java Framework. + +If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ . + +## Running the application in dev mode + +You can run your application in dev mode that enables live coding using: +```shell script +./gradlew quarkusDev +``` + +> **_NOTE:_** Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:8080/q/dev/. + +## Packaging and running the application + +The application can be packaged using: +```shell script +./gradlew build +``` +It produces the `quarkus-run.jar` file in the `build/quarkus-app/` directory. +Be aware that it’s not an _über-jar_ as the dependencies are copied into the `build/quarkus-app/lib/` directory. + +The application is now runnable using `java -jar build/quarkus-app/quarkus-run.jar`. + +If you want to build an _über-jar_, execute the following command: +```shell script +./gradlew build -Dquarkus.package.jar.type=uber-jar +``` + +The application, packaged as an _über-jar_, is now runnable using `java -jar build/*-runner.jar`. + +## Creating a native executable + +You can create a native executable using: +```shell script +./gradlew build -Dquarkus.native.enabled=true +``` + +Or, if you don't have GraalVM installed, you can run the native executable build in a container using: +```shell script +./gradlew build -Dquarkus.native.enabled=true -Dquarkus.native.container-build=true +``` + +You can then execute your native executable with: `./build/flow-jvm-0.0.1-SNAPSHOT-runner` + +If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling. + +## Provided Code + +### REST + +Easily start your REST Web Services + +[Related guide section...](https://quarkus.io/guides/getting-started-reactive#reactive-jax-rs-resources) diff --git a/flow-jvm/build.gradle b/flow-jvm/build.gradle new file mode 100644 index 0000000000..9ea6bdc5f7 --- /dev/null +++ b/flow-jvm/build.gradle @@ -0,0 +1,69 @@ +plugins { + id 'java' + id 'io.quarkus' +} + +repositories { + mavenCentral() + mavenLocal() +} + +ext { + icebergLibVersion = '1.4.2' +} + + +dependencies { + implementation 'io.quarkus:quarkus-config-yaml' + implementation enforcedPlatform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}") + implementation 'io.quarkus:quarkus-arc' + implementation 'io.quarkus:quarkus-grpc' + implementation 'io.quarkus:quarkus-rest' + + implementation "org.apache.iceberg:iceberg-core:${icebergLibVersion}" + implementation "org.apache.iceberg:iceberg-common:${icebergLibVersion}" + implementation "org.apache.iceberg:iceberg-data:${icebergLibVersion}" + implementation "org.apache.iceberg:iceberg-parquet:${icebergLibVersion}" + + implementation 'org.apache.avro:avro:1.11.3' + + implementation 'org.apache.hadoop:hadoop-client:3.4.0' + + // JDBC Drivers + runtimeOnly 'org.postgresql:postgresql:42.7.3' + + // AWS Dependencies + implementation "org.apache.iceberg:iceberg-aws:${icebergLibVersion}" + runtimeOnly 'software.amazon.awssdk:bundle:2.25.60' + + + testImplementation 'io.quarkus:quarkus-junit5' + testImplementation 'io.rest-assured:rest-assured' + +} + +group 'io.peerdb' +version '0.0.1-SNAPSHOT' + +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + +test { + systemProperty "java.util.logging.manager", "org.jboss.logmanager.LogManager" +} +compileJava { + options.encoding = 'UTF-8' + options.compilerArgs << '-parameters' +} + +compileTestJava { + options.encoding = 'UTF-8' +} + + + +quarkus { + quarkusBuildProperties.put("quarkus.grpc.codegen.proto-directory", "${project.projectDir}/../protos") +} diff --git a/flow-jvm/gradle.properties b/flow-jvm/gradle.properties new file mode 100644 index 0000000000..dde45ebcf2 --- /dev/null +++ b/flow-jvm/gradle.properties @@ -0,0 +1,7 @@ +#Gradle properties +#Fri May 24 04:59:17 IST 2024 +quarkusPlatformArtifactId=quarkus-bom +quarkusPlatformGroupId=io.quarkus.platform +quarkusPlatformVersion=3.10.2 +quarkusPluginId=io.quarkus +quarkusPluginVersion=3.10.2 diff --git a/flow-jvm/gradle/wrapper/gradle-wrapper.jar b/flow-jvm/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..62d4c053550b91381bbd28b1afc82d634bf73a8a GIT binary patch literal 58910 zcma&ObC74zk}X`WF59+k+qTVL*+!RbS9RI8Z5v&-ZFK4Nn|tqzcjwK__x+Iv5xL`> zj94dg?X`0sMHx^qXds{;KY)OMg#H>35XgTVfq6#vc9ww|9) z@UMfwUqk)B9p!}NrNqTlRO#i!ALOPcWo78-=iy}NsAr~T8T0X0%G{DhX~u-yEwc29WQ4D zuv2j{a&j?qB4wgCu`zOXj!~YpTNFg)TWoV>DhYlR^Gp^rkOEluvxkGLB?!{fD!T@( z%3cy>OkhbIKz*R%uoKqrg1%A?)uTZD&~ssOCUBlvZhx7XHQ4b7@`&sPdT475?*zWy z>xq*iK=5G&N6!HiZaD{NSNhWL;+>Quw_#ZqZbyglna!Fqn3N!$L`=;TFPrhodD-Q` z1l*=DP2gKJP@)cwI@-M}?M$$$%u~=vkeC%>cwR$~?y6cXx-M{=wdT4|3X(@)a|KkZ z`w$6CNS@5gWS7s7P86L<=vg$Mxv$?)vMj3`o*7W4U~*Nden}wz=y+QtuMmZ{(Ir1D zGp)ZsNiy{mS}Au5;(fYf93rs^xvi(H;|H8ECYdC`CiC&G`zw?@)#DjMc7j~daL_A$ z7e3nF2$TKlTi=mOftyFBt8*Xju-OY@2k@f3YBM)-v8+5_o}M?7pxlNn)C0Mcd@87?+AA4{Ti2ptnYYKGp`^FhcJLlT%RwP4k$ad!ho}-^vW;s{6hnjD0*c39k zrm@PkI8_p}mnT&5I@=O1^m?g}PN^8O8rB`;t`6H+?Su0IR?;8txBqwK1Au8O3BZAX zNdJB{bpQWR@J|e=Z>XSXV1DB{uhr3pGf_tb)(cAkp)fS7*Qv))&Vkbb+cvG!j}ukd zxt*C8&RN}5ck{jkw0=Q7ldUp0FQ&Pb_$M7a@^nf`8F%$ftu^jEz36d#^M8Ia{VaTy z5(h$I)*l3i!VpPMW+XGgzL~fcN?{~1QWu9!Gu0jOWWE zNW%&&by0DbXL&^)r-A*7R@;T$P}@3eOj#gqJ!uvTqBL5bupU91UK#d|IdxBUZAeh1 z>rAI#*Y4jv>uhOh7`S@mnsl0g@1C;k$Z%!d*n8#_$)l}-1&z2kr@M+xWoKR z!KySy-7h&Bf}02%JeXmQGjO3ntu={K$jy$rFwfSV8!zqAL_*&e2|CJ06`4&0+ceI026REfNT>JzAdwmIlKLEr2? zaZ#d*XFUN*gpzOxq)cysr&#6zNdDDPH% zd8_>3B}uA7;bP4fKVdd~Og@}dW#74ceETOE- zlZgQqQfEc?-5ly(Z5`L_CCM!&Uxk5#wgo=OLs-kFHFG*cTZ)$VE?c_gQUW&*!2@W2 z7Lq&_Kf88OCo?BHCtwe*&fu&8PQ(R5&lnYo8%+U73U)Ec2&|A)Y~m7(^bh299REPe zn#gyaJ4%o4>diN3z%P5&_aFUmlKytY$t21WGwx;3?UC}vlxi-vdEQgsKQ;=#sJ#ll zZeytjOad$kyON4XxC}frS|Ybh`Yq!<(IrlOXP3*q86ImyV*mJyBn$m~?#xp;EplcM z+6sez%+K}Xj3$YN6{}VL;BZ7Fi|iJj-ywlR+AP8lq~mnt5p_%VmN{Sq$L^z!otu_u znVCl@FgcVXo510e@5(wnko%Pv+^r^)GRh;>#Z(|#cLnu_Y$#_xG&nvuT+~gzJsoSi zBvX`|IS~xaold!`P!h(v|=>!5gk)Q+!0R1Ge7!WpRP{*Ajz$oGG$_?Ajvz6F0X?809o`L8prsJ*+LjlGfSziO;+ zv>fyRBVx#oC0jGK8$%$>Z;0+dfn8x;kHFQ?Rpi7(Rc{Uq{63Kgs{IwLV>pDK7yX-2 zls;?`h!I9YQVVbAj7Ok1%Y+F?CJa-Jl>1x#UVL(lpzBBH4(6v0^4 z3Tf`INjml5`F_kZc5M#^J|f%7Hgxg3#o}Zwx%4l9yYG!WaYUA>+dqpRE3nw#YXIX%= ziH3iYO~jr0nP5xp*VIa#-aa;H&%>{mfAPPlh5Fc!N7^{!z$;p-p38aW{gGx z)dFS62;V;%%fKp&i@+5x=Cn7Q>H`NofJGXmNeh{sOL+Nk>bQJJBw3K*H_$}%*xJM=Kh;s#$@RBR z|75|g85da@#qT=pD777m$wI!Q8SC4Yw3(PVU53bzzGq$IdGQoFb-c_(iA_~qD|eAy z@J+2!tc{|!8fF;%6rY9`Q!Kr>MFwEH%TY0y>Q(D}xGVJM{J{aGN0drG&|1xO!Ttdw z-1^gQ&y~KS5SeslMmoA$Wv$ly={f}f9<{Gm!8ycp*D9m*5Ef{ymIq!MU01*)#J1_! zM_i4{LYButqlQ>Q#o{~W!E_#(S=hR}kIrea_67Z5{W>8PD>g$f;dTvlD=X@T$8D0;BWkle@{VTd&D5^)U>(>g(jFt4lRV6A2(Te->ooI{nk-bZ(gwgh zaH4GT^wXPBq^Gcu%xW#S#p_&x)pNla5%S5;*OG_T^PhIIw1gXP&u5c;{^S(AC*+$> z)GuVq(FT@zq9;i{*9lEsNJZ)??BbSc5vF+Kdh-kL@`(`l5tB4P!9Okin2!-T?}(w% zEpbEU67|lU#@>DppToestmu8Ce=gz=e#V+o)v)#e=N`{$MI5P0O)_fHt1@aIC_QCv=FO`Qf=Ga%^_NhqGI)xtN*^1n{ z&vgl|TrKZ3Vam@wE0p{c3xCCAl+RqFEse@r*a<3}wmJl-hoJoN<|O2zcvMRl<#BtZ z#}-bPCv&OTw`GMp&n4tutf|er`@#d~7X+);##YFSJ)BitGALu}-N*DJdCzs(cQ?I- z6u(WAKH^NUCcOtpt5QTsQRJ$}jN28ZsYx+4CrJUQ%egH zo#tMoywhR*oeIkS%}%WUAIbM`D)R6Ya&@sZvvUEM7`fR0Ga03*=qaEGq4G7-+30Ck zRkje{6A{`ebq?2BTFFYnMM$xcQbz0nEGe!s%}O)m={`075R0N9KTZ>vbv2^eml>@}722%!r#6Wto}?vNst? zs`IasBtcROZG9+%rYaZe^=5y3chDzBf>;|5sP0!sP(t^= z^~go8msT@|rp8LJ8km?4l?Hb%o10h7(ixqV65~5Y>n_zG3AMqM3UxUNj6K-FUgMT7 z*Dy2Y8Ws+%`Z*~m9P zCWQ8L^kA2$rf-S@qHow$J86t)hoU#XZ2YK~9GXVR|*`f6`0&8j|ss_Ai-x=_;Df^*&=bW$1nc{Gplm zF}VF`w)`5A;W@KM`@<9Bw_7~?_@b{Z`n_A6c1AG#h#>Z$K>gX6reEZ*bZRjCup|0# zQ{XAb`n^}2cIwLTN%5Ix`PB*H^(|5S{j?BwItu+MS`1)VW=TnUtt6{3J!WR`4b`LW z?AD#ZmoyYpL=903q3LSM=&5eNP^dwTDRD~iP=}FXgZ@2WqfdyPYl$9do?wX{RU*$S zgQ{OqXK-Yuf4+}x6P#A*la&^G2c2TC;aNNZEYuB(f25|5eYi|rd$;i0qk7^3Ri8of ziP~PVT_|4$n!~F-B1_Et<0OJZ*e+MN;5FFH`iec(lHR+O%O%_RQhvbk-NBQ+$)w{D+dlA0jxI;z|P zEKW`!X)${xzi}Ww5G&@g0akBb_F`ziv$u^hs0W&FXuz=Ap>SUMw9=M?X$`lgPRq11 zqq+n44qL;pgGO+*DEc+Euv*j(#%;>p)yqdl`dT+Og zZH?FXXt`<0XL2@PWYp|7DWzFqxLK)yDXae&3P*#+f+E{I&h=$UPj;ey9b`H?qe*Oj zV|-qgI~v%&oh7rzICXfZmg$8$B|zkjliQ=e4jFgYCLR%yi!9gc7>N z&5G#KG&Hr+UEfB;M(M>$Eh}P$)<_IqC_WKOhO4(cY@Gn4XF(#aENkp&D{sMQgrhDT zXClOHrr9|POHqlmm+*L6CK=OENXbZ+kb}t>oRHE2xVW<;VKR@ykYq04LM9L-b;eo& zl!QQo!Sw{_$-qosixZJWhciN>Gbe8|vEVV2l)`#5vKyrXc6E`zmH(76nGRdL)pqLb@j<&&b!qJRLf>d`rdz}^ZSm7E;+XUJ ziy;xY&>LM?MA^v0Fu8{7hvh_ynOls6CI;kQkS2g^OZr70A}PU;i^~b_hUYN1*j-DD zn$lHQG9(lh&sDii)ip*{;Sb_-Anluh`=l~qhqbI+;=ZzpFrRp&T+UICO!OoqX@Xr_ z32iJ`xSpx=lDDB_IG}k+GTYG@K8{rhTS)aoN8D~Xfe?ul&;jv^E;w$nhu-ICs&Q)% zZ=~kPNZP0-A$pB8)!`TEqE`tY3Mx^`%O`?EDiWsZpoP`e-iQ#E>fIyUx8XN0L z@S-NQwc;0HjSZKWDL}Au_Zkbh!juuB&mGL0=nO5)tUd_4scpPy&O7SNS^aRxUy0^< zX}j*jPrLP4Pa0|PL+nrbd4G;YCxCK-=G7TG?dby~``AIHwxqFu^OJhyIUJkO0O<>_ zcpvg5Fk$Wpj}YE3;GxRK67P_Z@1V#+pu>pRj0!mFf(m_WR3w3*oQy$s39~U7Cb}p(N&8SEwt+)@%o-kW9Ck=^?tvC2$b9% ze9(Jn+H`;uAJE|;$Flha?!*lJ0@lKfZM>B|c)3lIAHb;5OEOT(2453m!LgH2AX=jK zQ93An1-#l@I@mwB#pLc;M7=u6V5IgLl>E%gvE|}Hvd4-bE1>gs(P^C}gTv*&t>W#+ zASLRX$y^DD3Jrht zwyt`yuA1j(TcP*0p*Xkv>gh+YTLrcN_HuaRMso~0AJg`^nL#52dGBzY+_7i)Ud#X) zVwg;6$WV20U2uyKt8<)jN#^1>PLg`I`@Mmut*Zy!c!zshSA!e^tWVoKJD%jN&ml#{ z@}B$j=U5J_#rc%T7(DGKF+WwIblEZ;Vq;CsG~OKxhWYGJx#g7fxb-_ya*D0=_Ys#f zhXktl=Vnw#Z_neW>Xe#EXT(4sT^3p6srKby4Ma5LLfh6XrHGFGgM;5Z}jv-T!f~=jT&n>Rk z4U0RT-#2fsYCQhwtW&wNp6T(im4dq>363H^ivz#>Sj;TEKY<)dOQU=g=XsLZhnR>e zd}@p1B;hMsL~QH2Wq>9Zb; zK`0`09fzuYg9MLJe~cdMS6oxoAD{kW3sFAqDxvFM#{GpP^NU@9$d5;w^WgLYknCTN z0)N425mjsJTI@#2kG-kB!({*+S(WZ-{SckG5^OiyP%(6DpRsx60$H8M$V65a_>oME z^T~>oG7r!ew>Y)&^MOBrgc-3PezgTZ2xIhXv%ExMFgSf5dQbD=Kj*!J4k^Xx!Z>AW ziZfvqJvtm|EXYsD%A|;>m1Md}j5f2>kt*gngL=enh<>#5iud0dS1P%u2o+>VQ{U%(nQ_WTySY(s#~~> zrTsvp{lTSup_7*Xq@qgjY@1#bisPCRMMHnOL48qi*jQ0xg~TSW%KMG9zN1(tjXix()2$N}}K$AJ@GUth+AyIhH6Aeh7qDgt#t*`iF5#A&g4+ zWr0$h9Zx6&Uo2!Ztcok($F>4NA<`dS&Js%L+67FT@WmI)z#fF~S75TUut%V($oUHw z$IJsL0X$KfGPZYjB9jaj-LaoDD$OMY4QxuQ&vOGo?-*9@O!Nj>QBSA6n$Lx|^ zky)4+sy{#6)FRqRt6nM9j2Lzba!U;aL%ZcG&ki1=3gFx6(&A3J-oo|S2_`*w9zT)W z4MBOVCp}?4nY)1))SOX#6Zu0fQQ7V{RJq{H)S#;sElY)S)lXTVyUXTepu4N)n85Xo zIpWPT&rgnw$D2Fsut#Xf-hO&6uA0n~a;a3!=_!Tq^TdGE&<*c?1b|PovU}3tfiIUu z){4W|@PY}zJOXkGviCw^x27%K_Fm9GuKVpd{P2>NJlnk^I|h2XW0IO~LTMj>2<;S* zZh2uRNSdJM$U$@=`zz}%;ucRx{aKVxxF7?0hdKh6&GxO6f`l2kFncS3xu0Ly{ew0& zeEP*#lk-8-B$LD(5yj>YFJ{yf5zb41PlW7S{D9zC4Aa4nVdkDNH{UsFJp)q-`9OYt zbOKkigbmm5hF?tttn;S4g^142AF^`kiLUC?e7=*JH%Qe>uW=dB24NQa`;lm5yL>Dyh@HbHy-f%6Vz^ zh&MgwYsh(z#_fhhqY$3*f>Ha}*^cU-r4uTHaT?)~LUj5``FcS46oyoI5F3ZRizVD% zPFY(_S&5GN8$Nl2=+YO6j4d|M6O7CmUyS&}m4LSn6}J`$M0ZzT&Ome)ZbJDFvM&}A zZdhDn(*viM-JHf84$!I(8eakl#zRjJH4qfw8=60 z11Ely^FyXjVvtv48-Fae7p=adlt9_F^j5#ZDf7)n!#j?{W?@j$Pi=k`>Ii>XxrJ?$ z^bhh|X6qC8d{NS4rX5P!%jXy=>(P+r9?W(2)|(=a^s^l~x*^$Enw$~u%WRuRHHFan{X|S;FD(Mr z@r@h^@Bs#C3G;~IJMrERd+D!o?HmFX&#i|~q(7QR3f8QDip?ms6|GV_$86aDb|5pc?_-jo6vmWqYi{P#?{m_AesA4xX zi&ki&lh0yvf*Yw~@jt|r-=zpj!bw<6zI3Aa^Wq{|*WEC}I=O!Re!l~&8|Vu<$yZ1p zs-SlwJD8K!$(WWyhZ+sOqa8cciwvyh%zd`r$u;;fsHn!hub0VU)bUv^QH?x30#;tH zTc_VbZj|prj7)d%ORU;Vs{#ERb>K8>GOLSImnF7JhR|g$7FQTU{(a7RHQ*ii-{U3X z^7+vM0R$8b3k1aSU&kxvVPfOz3~)0O2iTYinV9_5{pF18j4b{o`=@AZIOAwwedB2@ ztXI1F04mg{<>a-gdFoRjq$6#FaevDn$^06L)k%wYq03&ysdXE+LL1#w$rRS1Y;BoS zH1x}{ms>LHWmdtP(ydD!aRdAa(d@csEo z0EF9L>%tppp`CZ2)jVb8AuoYyu;d^wfje6^n6`A?6$&%$p>HcE_De-Zh)%3o5)LDa zskQ}%o7?bg$xUj|n8gN9YB)z!N&-K&!_hVQ?#SFj+MpQA4@4oq!UQ$Vm3B`W_Pq3J z=ngFP4h_y=`Iar<`EESF9){%YZVyJqLPGq07TP7&fSDmnYs2NZQKiR%>){imTBJth zPHr@p>8b+N@~%43rSeNuOz;rgEm?14hNtI|KC6Xz1d?|2J`QS#`OW7gTF_;TPPxu@ z)9J9>3Lx*bc>Ielg|F3cou$O0+<b34_*ZJhpS&$8DP>s%47a)4ZLw`|>s=P_J4u z?I_%AvR_z8of@UYWJV?~c4Yb|A!9n!LEUE6{sn@9+D=0w_-`szJ_T++x3MN$v-)0d zy`?1QG}C^KiNlnJBRZBLr4G~15V3$QqC%1G5b#CEB0VTr#z?Ug%Jyv@a`QqAYUV~^ zw)d|%0g&kl{j#FMdf$cn(~L@8s~6eQ)6{`ik(RI(o9s0g30Li{4YoxcVoYd+LpeLz zai?~r)UcbYr@lv*Z>E%BsvTNd`Sc?}*}>mzJ|cr0Y(6rA7H_6&t>F{{mJ^xovc2a@ zFGGDUcGgI-z6H#o@Gj29C=Uy{wv zQHY2`HZu8+sBQK*_~I-_>fOTKEAQ8_Q~YE$c?cSCxI;vs-JGO`RS464Ft06rpjn+a zqRS0Y3oN(9HCP@{J4mOWqIyD8PirA!pgU^Ne{LHBG;S*bZpx3|JyQDGO&(;Im8!ed zNdpE&?3U?E@O~>`@B;oY>#?gXEDl3pE@J30R1;?QNNxZ?YePc)3=NS>!STCrXu*lM z69WkLB_RBwb1^-zEm*tkcHz3H;?v z;q+x0Jg$|?5;e1-kbJnuT+^$bWnYc~1qnyVTKh*cvM+8yJT-HBs1X@cD;L$su65;i z2c1MxyL~NuZ9+)hF=^-#;dS#lFy^Idcb>AEDXu1!G4Kd8YPy~0lZz$2gbv?su}Zn} zGtIbeYz3X8OA9{sT(aleold_?UEV{hWRl(@)NH6GFH@$<8hUt=dNte%e#Jc>7u9xi zuqv!CRE@!fmZZ}3&@$D>p0z=*dfQ_=IE4bG0hLmT@OP>x$e`qaqf_=#baJ8XPtOpWi%$ep1Y)o2(sR=v)M zt(z*pGS$Z#j_xq_lnCr+x9fwiT?h{NEn#iK(o)G&Xw-#DK?=Ms6T;%&EE${Gq_%99 z6(;P~jPKq9llc+cmI(MKQ6*7PcL)BmoI}MYFO)b3-{j>9FhNdXLR<^mnMP`I7z0v` zj3wxcXAqi4Z0kpeSf>?V_+D}NULgU$DBvZ^=0G8Bypd7P2>;u`yW9`%4~&tzNJpgp zqB+iLIM~IkB;ts!)exn643mAJ8-WlgFE%Rpq!UMYtB?$5QAMm)%PT0$$2{>Yu7&U@ zh}gD^Qdgu){y3ANdB5{75P;lRxSJPSpQPMJOiwmpMdT|?=q;&$aTt|dl~kvS z+*i;6cEQJ1V`R4Fd>-Uzsc=DPQ7A7#VPCIf!R!KK%LM&G%MoZ0{-8&99H!|UW$Ejv zhDLX3ESS6CgWTm#1ZeS2HJb`=UM^gsQ84dQpX(ESWSkjn>O zVxg%`@mh(X9&&wN$lDIc*@>rf?C0AD_mge3f2KkT6kGySOhXqZjtA?5z`vKl_{(5g z&%Y~9p?_DL{+q@siT~*3Q*$nWXQfNN;%s_eHP_A;O`N`SaoB z6xYR;z_;HQ2xAa9xKgx~2f2xEKiEDpGPH1d@||v#f#_Ty6_gY>^oZ#xac?pc-F`@ z*}8sPV@xiz?efDMcmmezYVw~qw=vT;G1xh+xRVBkmN66!u(mRG3G6P#v|;w@anEh7 zCf94arw%YB*=&3=RTqX?z4mID$W*^+&d6qI*LA-yGme;F9+wTsNXNaX~zl2+qIK&D-aeN4lr0+yP;W>|Dh?ms_ogT{DT+ ztXFy*R7j4IX;w@@R9Oct5k2M%&j=c_rWvoul+` z<18FH5D@i$P38W9VU2(EnEvlJ(SHCqTNBa)brkIjGP|jCnK&Qi%97tikU}Y#3L?s! z2ujL%YiHO-#!|g5066V01hgT#>fzls7P>+%D~ogOT&!Whb4iF=CnCto82Yb#b`YoVsj zS2q^W0Rj!RrM@=_GuPQy5*_X@Zmu`TKSbqEOP@;Ga&Rrr>#H@L41@ZX)LAkbo{G8+ z;!5EH6vv-ip0`tLB)xUuOX(*YEDSWf?PIxXe`+_B8=KH#HFCfthu}QJylPMTNmoV; zC63g%?57(&osaH^sxCyI-+gwVB|Xs2TOf=mgUAq?V~N_5!4A=b{AXbDae+yABuuu3B_XSa4~c z1s-OW>!cIkjwJf4ZhvT|*IKaRTU)WAK=G|H#B5#NB9<{*kt?7`+G*-^<)7$Iup@Um z7u*ABkG3F*Foj)W9-I&@BrN8(#$7Hdi`BU#SR1Uz4rh&=Ey!b76Qo?RqBJ!U+rh(1 znw@xw5$)4D8OWtB_^pJO*d~2Mb-f~>I!U#*=Eh*xa6$LX?4Evp4%;ENQR!mF4`f7F zpG!NX=qnCwE8@NAbQV`*?!v0;NJ(| zBip8}VgFVsXFqslXUV>_Z>1gmD(7p#=WACXaB|Y`=Kxa=p@_ALsL&yAJ`*QW^`2@% zW7~Yp(Q@ihmkf{vMF?kqkY%SwG^t&CtfRWZ{syK@W$#DzegcQ1>~r7foTw3^V1)f2Tq_5f$igmfch;8 zT-<)?RKcCdQh6x^mMEOS;4IpQ@F2q-4IC4%*dU@jfHR4UdG>Usw4;7ESpORL|2^#jd+@zxz{(|RV*1WKrw-)ln*8LnxVkKDfGDHA%7`HaiuvhMu%*mY9*Ya{Ti#{DW?i0 zXXsp+Bb(_~wv(3t70QU3a$*<$1&zm1t++x#wDLCRI4K)kU?Vm9n2c0m@TyUV&&l9%}fulj!Z9)&@yIcQ3gX}l0b1LbIh4S z5C*IDrYxR%qm4LVzSk{0;*npO_SocYWbkAjA6(^IAwUnoAzw_Uo}xYFo?Y<-4Zqec z&k7HtVlFGyt_pA&kX%P8PaRD8y!Wsnv}NMLNLy-CHZf(ObmzV|t-iC#@Z9*d-zUsx zxcYWw{H)nYXVdnJu5o-U+fn~W z-$h1ax>h{NlWLA7;;6TcQHA>UJB$KNk74T1xNWh9)kwK~wX0m|Jo_Z;g;>^E4-k4R zRj#pQb-Hg&dAh}*=2;JY*aiNZzT=IU&v|lQY%Q|=^V5pvTR7^t9+@+ST&sr!J1Y9a z514dYZn5rg6@4Cy6P`-?!3Y& z?B*5zw!mTiD2)>f@3XYrW^9V-@%YFkE_;PCyCJ7*?_3cR%tHng9%ZpIU}LJM=a+0s z(SDDLvcVa~b9O!cVL8)Q{d^R^(bbG=Ia$)dVN_tGMee3PMssZ7Z;c^Vg_1CjZYTnq z)wnF8?=-MmqVOMX!iE?YDvHCN?%TQtKJMFHp$~kX4}jZ;EDqP$?jqJZjoa2PM@$uZ zF4}iab1b5ep)L;jdegC3{K4VnCH#OV;pRcSa(&Nm50ze-yZ8*cGv;@+N+A?ncc^2z9~|(xFhwOHmPW@ zR5&)E^YKQj@`g=;zJ_+CLamsPuvppUr$G1#9urUj+p-mPW_QSSHkPMS!52t>Hqy|g z_@Yu3z%|wE=uYq8G>4`Q!4zivS}+}{m5Zjr7kMRGn_p&hNf|pc&f9iQ`^%78rl#~8 z;os@rpMA{ZioY~(Rm!Wf#Wx##A0PthOI341QiJ=G*#}pDAkDm+{0kz&*NB?rC0-)glB{0_Tq*^o zVS1>3REsv*Qb;qg!G^9;VoK)P*?f<*H&4Su1=}bP^Y<2PwFpoqw#up4IgX3L z`w~8jsFCI3k~Y9g(Y9Km`y$0FS5vHb)kb)Jb6q-9MbO{Hbb zxg?IWQ1ZIGgE}wKm{axO6CCh~4DyoFU+i1xn#oyfe+<{>=^B5tm!!*1M?AW8c=6g+%2Ft97_Hq&ZmOGvqGQ!Bn<_Vw`0DRuDoB6q8ME<;oL4kocr8E$NGoLI zXWmI7Af-DR|KJw!vKp2SI4W*x%A%5BgDu%8%Iato+pWo5`vH@!XqC!yK}KLzvfS(q z{!y(S-PKbk!qHsgVyxKsQWk_8HUSSmslUA9nWOjkKn0%cwn%yxnkfxn?Y2rysXKS=t-TeI%DN$sQ{lcD!(s>(4y#CSxZ4R} zFDI^HPC_l?uh_)-^ppeYRkPTPu~V^0Mt}#jrTL1Q(M;qVt4zb(L|J~sxx7Lva9`mh zz!#A9tA*6?q)xThc7(gB2Ryam$YG4qlh00c}r&$y6u zIN#Qxn{7RKJ+_r|1G1KEv!&uKfXpOVZ8tK{M775ws%nDyoZ?bi3NufNbZs)zqXiqc zqOsK@^OnlFMAT&mO3`@3nZP$3lLF;ds|;Z{W(Q-STa2>;)tjhR17OD|G>Q#zJHb*> zMO<{WIgB%_4MG0SQi2;%f0J8l_FH)Lfaa>*GLobD#AeMttYh4Yfg22@q4|Itq};NB z8;o*+@APqy@fPgrc&PTbGEwdEK=(x5K!If@R$NiO^7{#j9{~w=RBG)ZkbOw@$7Nhl zyp{*&QoVBd5lo{iwl2gfyip@}IirZK;ia(&ozNl!-EEYc=QpYH_= zJkv7gA{!n4up6$CrzDJIBAdC7D5D<_VLH*;OYN>_Dx3AT`K4Wyx8Tm{I+xplKP6k7 z2sb!i7)~%R#J0$|hK?~=u~rnH7HCUpsQJujDDE*GD`qrWWog+C+E~GGy|Hp_t4--} zrxtrgnPh}r=9o}P6jpAQuDN}I*GI`8&%Lp-C0IOJt#op)}XSr!ova@w{jG2V=?GXl3zEJJFXg)U3N>BQP z*Lb@%Mx|Tu;|u>$-K(q^-HG!EQ3o93%w(A7@ngGU)HRWoO&&^}U$5x+T&#zri>6ct zXOB#EF-;z3j311K`jrYyv6pOPF=*`SOz!ack=DuEi({UnAkL5H)@R?YbRKAeP|06U z?-Ns0ZxD0h9D8)P66Sq$w-yF+1hEVTaul%&=kKDrQtF<$RnQPZ)ezm1`aHIjAY=!S z`%vboP`?7mItgEo4w50C*}Ycqp9_3ZEr^F1;cEhkb`BNhbc6PvnXu@wi=AoezF4~K zkxx%ps<8zb=wJ+9I8o#do)&{(=yAlNdduaDn!=xGSiuo~fLw~Edw$6;l-qaq#Z7?# zGrdU(Cf-V@$x>O%yRc6!C1Vf`b19ly;=mEu8u9|zitcG^O`lbNh}k=$%a)UHhDwTEKis2yc4rBGR>l*(B$AC7ung&ssaZGkY-h(fpwcPyJSx*9EIJMRKbMP9}$nVrh6$g-Q^5Cw)BeWqb-qi#37ZXKL!GR;ql)~ z@PP*-oP?T|ThqlGKR84zi^CN z4TZ1A)7vL>ivoL2EU_~xl-P{p+sE}9CRwGJDKy{>0KP+gj`H9C+4fUMPnIB1_D`A- z$1`G}g0lQmqMN{Y&8R*$xYUB*V}dQPxGVZQ+rH!DVohIoTbh%#z#Tru%Px@C<=|og zGDDwGq7yz`%^?r~6t&>x*^We^tZ4!E4dhwsht#Pb1kCY{q#Kv;z%Dp#Dq;$vH$-(9 z8S5tutZ}&JM2Iw&Y-7KY4h5BBvS=Ove0#+H2qPdR)WyI zYcj)vB=MA{7T|3Ij_PN@FM@w(C9ANBq&|NoW30ccr~i#)EcH)T^3St~rJ0HKKd4wr z@_+132;Bj+>UC@h)Ap*8B4r5A1lZ!Dh%H7&&hBnlFj@eayk=VD*i5AQc z$uN8YG#PL;cuQa)Hyt-}R?&NAE1QT>svJDKt*)AQOZAJ@ zyxJoBebiobHeFlcLwu_iI&NEZuipnOR;Tn;PbT1Mt-#5v5b*8ULo7m)L-eti=UcGf zRZXidmxeFgY!y80-*PH-*=(-W+fK%KyUKpg$X@tuv``tXj^*4qq@UkW$ZrAo%+hay zU@a?z&2_@y)o@D!_g>NVxFBO!EyB&6Z!nd4=KyDP^hl!*(k{dEF6@NkXztO7gIh zQ&PC+p-8WBv;N(rpfKdF^@Z~|E6pa)M1NBUrCZvLRW$%N%xIbv^uv?=C!=dDVq3%* zgvbEBnG*JB*@vXx8>)7XL*!{1Jh=#2UrByF7U?Rj_}VYw88BwqefT_cCTv8aTrRVjnn z1HNCF=44?*&gs2`vCGJVHX@kO z240eo#z+FhI0=yy6NHQwZs}a+J~4U-6X`@ zZ7j+tb##m`x%J66$a9qXDHG&^kp|GkFFMmjD(Y-k_ClY~N$H|n@NkSDz=gg?*2ga5 z)+f)MEY>2Lp15;~o`t`qj;S>BaE;%dv@Ux11yq}I(k|o&`5UZFUHn}1kE^gIK@qV& z!S2IhyU;->VfA4Qb}m7YnkIa9%z{l~iPWo2YPk-`hy2-Eg=6E$21plQA5W2qMZDFU z-a-@Dndf%#on6chT`dOKnU9}BJo|kJwgGC<^nfo34zOKH96LbWY7@Wc%EoFF=}`VU zksP@wd%@W;-p!e^&-)N7#oR331Q)@9cx=mOoU?_Kih2!Le*8fhsZ8Qvo6t2vt+UOZ zw|mCB*t2%z21YqL>whu!j?s~}-L`OS+jdg1(XnmYw$rg~r(?5Y+qTg`$F}q3J?GtL z@BN&8#`u2RqkdG4yGGTus@7U_%{6C{XAhFE!2SelH?KtMtX@B1GBhEIDL-Bj#~{4! zd}p7!#XE9Lt;sy@p5#Wj*jf8zGv6tTotCR2X$EVOOup;GnRPRVU5A6N@Lh8?eA7k? zn~hz&gY;B0ybSpF?qwQ|sv_yO=8}zeg2$0n3A8KpE@q26)?707pPw?H76lCpjp=5r z6jjp|auXJDnW}uLb6d7rsxekbET9(=zdTqC8(F5@NNqII2+~yB;X5iJNQSiv`#ozm zf&p!;>8xAlwoxUC3DQ#!31ylK%VrcwS<$WeCY4V63V!|221oj+5#r}fGFQ}|uwC0) zNl8(CF}PD`&Sj+p{d!B&&JtC+VuH z#>US`)YQrhb6lIAYb08H22y(?)&L8MIQsA{26X`R5Km{YU)s!x(&gIsjDvq63@X`{ z=7{SiH*_ZsPME#t2m|bS76Uz*z{cpp1m|s}HIX}Ntx#v7Eo!1%G9__4dGSGl`p+xi zZ!VK#Qe;Re=9bqXuW+0DSP{uZ5-QXrNn-7qW19K0qU}OhVru7}3vqsG?#D67 zb}crN;QwsH*vymw(maZr_o|w&@sQki(X+D)gc5Bt&@iXisFG;eH@5d43~Wxq|HO(@ zV-rip4n#PEkHCWCa5d?@cQp^B;I-PzOfag|t-cuvTapQ@MWLmh*41NH`<+A+JGyKX zyYL6Ba7qqa5j@3lOk~`OMO7f0!@FaOeZxkbG@vXP(t3#U*fq8=GAPqUAS>vW2uxMk{a(<0=IxB;# zMW;M+owrHaZBp`3{e@7gJCHP!I(EeyGFF;pdFPdeP+KphrulPSVidmg#!@W`GpD&d z9p6R`dpjaR2E1Eg)Ws{BVCBU9-aCgN57N~uLvQZH`@T+2eOBD%73rr&sV~m#2~IZx zY_8f8O;XLu2~E3JDXnGhFvsyb^>*!D>5EtlKPe%kOLv6*@=Jpci`8h0z?+fbBUg_7 zu6DjqO=$SjAv{|Om5)nz41ZkS4E_|fk%NDY509VV5yNeo%O|sb>7C#wj8mL9cEOFh z>nDz%?vb!h*!0dHdnxDA>97~EoT~!N40>+)G2CeYdOvJr5^VnkGz)et&T9hrD(VAgCAJjQ7V$O?csICB*HFd^k@$M5*v$PZJD-OVL?Ze(U=XGqZPVG8JQ z<~ukO%&%nNXYaaRibq#B1KfW4+XMliC*Tng2G(T1VvP;2K~;b$EAqthc${gjn_P!b zs62UT(->A>!ot}cJXMZHuy)^qfqW~xO-In2);e>Ta{LD6VG2u&UT&a@>r-;4<)cJ9 zjpQThb4^CY)Ev0KR7TBuT#-v}W?Xzj{c7$S5_zJA57Qf=$4^npEjl9clH0=jWO8sX z3Fuu0@S!WY>0XX7arjH`?)I<%2|8HfL!~#c+&!ZVmhbh`wbzy0Ux|Jpy9A{_7GGB0 zadZ48dW0oUwUAHl%|E-Q{gA{z6TXsvU#Hj09<7i)d}wa+Iya)S$CVwG{4LqtB>w%S zKZx(QbV7J9pYt`W4+0~f{hoo5ZG<0O&&5L57oF%hc0xGJ@Zrg_D&lNO=-I^0y#3mxCSZFxN2-tN_mU@7<@PnWG?L5OSqkm8TR!`| zRcTeWH~0z1JY^%!N<(TtxSP5^G9*Vw1wub`tC-F`=U)&sJVfvmh#Pi`*44kSdG};1 zJbHOmy4Ot|%_?@$N?RA9fF?|CywR8Sf(SCN_luM8>(u0NSEbKUy7C(Sk&OuWffj)f za`+mo+kM_8OLuCUiA*CNE|?jra$M=$F3t+h-)?pXz&r^F!ck;r##`)i)t?AWq-9A9 zSY{m~TC1w>HdEaiR*%j)L);H{IULw)uxDO>#+WcBUe^HU)~L|9#0D<*Ld459xTyew zbh5vCg$a>`RCVk)#~ByCv@Ce!nm<#EW|9j><#jQ8JfTmK#~jJ&o0Fs9jz0Ux{svdM4__<1 zrb>H(qBO;v(pXPf5_?XDq!*3KW^4>(XTo=6O2MJdM^N4IIcYn1sZZpnmMAEdt}4SU zPO54j2d|(xJtQ9EX-YrlXU1}6*h{zjn`in-N!Ls}IJsG@X&lfycsoCemt_Ym(PXhv zc*QTnkNIV=Ia%tg%pwJtT^+`v8ng>;2~ps~wdqZSNI7+}-3r+#r6p`8*G;~bVFzg= z!S3&y)#iNSUF6z;%o)%h!ORhE?CUs%g(k2a-d576uOP2@QwG-6LT*G!I$JQLpd`cz z-2=Brr_+z96a0*aIhY2%0(Sz=|D`_v_7h%Yqbw2)8@1DwH4s*A82krEk{ zoa`LbCdS)R?egRWNeHV8KJG0Ypy!#}kslun?67}^+J&02!D??lN~t@;h?GS8#WX`)6yC**~5YNhN_Hj}YG<%2ao^bpD8RpgV|V|GQwlL27B zEuah|)%m1s8C6>FLY0DFe9Ob66fo&b8%iUN=y_Qj;t3WGlNqP9^d#75ftCPA*R4E8 z)SWKBKkEzTr4JqRMEs`)0;x8C35yRAV++n(Cm5++?WB@ya=l8pFL`N0ag`lWhrYo3 zJJ$< zQ*_YAqIGR*;`VzAEx1Pd4b3_oWtdcs7LU2#1#Ls>Ynvd8k^M{Ef?8`RxA3!Th-?ui{_WJvhzY4FiPxA?E4+NFmaC-Uh*a zeLKkkECqy>Qx&1xxEhh8SzMML=8VP}?b*sgT9ypBLF)Zh#w&JzP>ymrM?nnvt!@$2 zh>N$Q>mbPAC2kNd&ab;FkBJ}39s*TYY0=@e?N7GX>wqaM>P=Y12lciUmve_jMF0lY zBfI3U2{33vWo(DiSOc}!5##TDr|dgX1Uojq9!vW3$m#zM_83EGsP6&O`@v-PDdO3P z>#!BEbqpOXd5s?QNnN!p+92SHy{sdpePXHL{d@c6UilT<#~I!tH$S(~o}c#(j<2%! zQvm}MvAj-95Ekx3D4+|e%!?lO(F+DFw9bxb-}rsWQl)b44###eUg4N?N-P(sFH2hF z`{zu?LmAxn2=2wCE8?;%ZDi#Y;Fzp+RnY8fWlzVz_*PDO6?Je&aEmuS>=uCXgdP6r zoc_JB^TA~rU5*geh{G*gl%_HnISMS~^@{@KVC;(aL^ZA-De+1zwUSXgT>OY)W?d6~ z72znET0m`53q%AVUcGraYxIcAB?OZA8AT!uK8jU+=t;WneL~|IeQ>$*dWa#x%rB(+ z5?xEkZ&b{HsZ4Ju9TQ|)c_SIp`7r2qMJgaglfSBHhl)QO1aNtkGr0LUn{@mvAt=}nd7#>7ru}&I)FNsa*x?Oe3-4G`HcaR zJ}c%iKlwh`x)yX1vBB;-Nr=7>$~(u=AuPX2#&Eh~IeFw%afU+U)td0KC!pHd zyn+X$L|(H3uNit-bpn7%G%{&LsAaEfEsD?yM<;U2}WtD4KuVKuX=ec9X zIe*ibp1?$gPL7<0uj*vmj2lWKe`U(f9E{KVbr&q*RsO;O>K{i-7W)8KG5~~uS++56 zm@XGrX@x+lGEjDQJp~XCkEyJG5Y57omJhGN{^2z5lj-()PVR&wWnDk2M?n_TYR(gM zw4kQ|+i}3z6YZq8gVUN}KiYre^sL{ynS}o{z$s&I z{(rWaLXxcQ=MB(Cz7W$??Tn*$1y(7XX)tv;I-{7F$fPB%6YC7>-Dk#=Y8o1=&|>t5 zV_VVts>Eb@)&4%m}!K*WfLoLl|3FW)V~E1Z!yu`Sn+bAP5sRDyu7NEbLt?khAyz-ZyL-}MYb&nQ zU16f@q7E1rh!)d%f^tTHE3cVoa%Xs%rKFc|temN1sa)aSlT*)*4k?Z>b3NP(IRXfq zlB^#G6BDA1%t9^Nw1BD>lBV(0XW5c?l%vyB3)q*;Z5V~SU;HkN;1kA3Nx!$!9wti= zB8>n`gt;VlBt%5xmDxjfl0>`K$fTU-C6_Z;!A_liu0@Os5reMLNk;jrlVF^FbLETI zW+Z_5m|ozNBn7AaQ<&7zk}(jmEdCsPgmo%^GXo>YYt82n&7I-uQ%A;k{nS~VYGDTn zlr3}HbWQG6xu8+bFu^9%%^PYCbkLf=*J|hr>Sw+#l(Y#ZGKDufa#f-f0k-{-XOb4i zwVG1Oa0L2+&(u$S7TvedS<1m45*>a~5tuOZ;3x%!f``{=2QQlJk|b4>NpD4&L+xI+ z+}S(m3}|8|Vv(KYAGyZK5x*sgwOOJklN0jsq|BomM>OuRDVFf_?cMq%B*iQ*&|vS9 zVH7Kh)SjrCBv+FYAE=$0V&NIW=xP>d-s7@wM*sdfjVx6-Y@=~>rz%2L*rKp|*WXIz z*vR^4tV&7MQpS9%{9b*>E9d_ls|toL7J|;srnW{l-}1gP_Qr-bBHt=}PL@WlE|&KH zCUmDLZb%J$ZzNii-5VeygOM?K8e$EcK=z-hIk63o4y63^_*RdaitO^THC{boKstphXZ2Z+&3ToeLQUG(0Frs?b zCxB+65h7R$+LsbmL51Kc)pz_`YpGEzFEclzb=?FJ=>rJwgcp0QH-UuKRS1*yCHsO) z-8t?Zw|6t($Eh&4K+u$I7HqVJBOOFCRcmMMH};RX_b?;rnk`rz@vxT_&|6V@q0~Uk z9ax|!pA@Lwn8h7syrEtDluZ6G!;@=GL> zse#PRQrdDs=qa_v@{Wv(3YjYD0|qocDC;-F~&{oaTP?@pi$n z1L6SlmFU2~%)M^$@C(^cD!y)-2SeHo3t?u3JiN7UBa7E2 z;<+_A$V084@>&u)*C<4h7jw9joHuSpVsy8GZVT;(>lZ(RAr!;)bwM~o__Gm~exd`K zKEgh2)w?ReH&syI`~;Uo4`x4$&X+dYKI{e`dS~bQuS|p zA`P_{QLV3r$*~lb=9vR^H0AxK9_+dmHX}Y} zIV*#65%jRWem5Z($ji{!6ug$En4O*=^CiG=K zp4S?+xE|6!cn$A%XutqNEgUqYY3fw&N(Z6=@W6*bxdp~i_yz5VcgSj=lf-6X1Nz75 z^DabwZ4*70$$8NsEy@U^W67tcy7^lNbu;|kOLcJ40A%J#pZe0d#n zC{)}+p+?8*ftUlxJE*!%$`h~|KZSaCb=jpK3byAcuHk7wk@?YxkT1!|r({P*KY^`u z!hw#`5$JJZGt@nkBK_nwWA31_Q9UGvv9r-{NU<&7HHMQsq=sn@O?e~fwl20tnSBG* zO%4?Ew6`aX=I5lqmy&OkmtU}bH-+zvJ_CFy z_nw#!8Rap5Wcex#5}Ldtqhr_Z$}@jPuYljTosS1+WG+TxZ>dGeT)?ZP3#3>sf#KOG z0)s%{cEHBkS)019}-1A2kd*it>y65-C zh7J9zogM74?PU)0c0YavY7g~%j%yiWEGDb+;Ew5g5Gq@MpVFFBNOpu0x)>Yn>G6uo zKE%z1EhkG_N5$a8f6SRm(25iH#FMeaJ1^TBcBy<04ID47(1(D)q}g=_6#^V@yI?Y&@HUf z`;ojGDdsvRCoTmasXndENqfWkOw=#cV-9*QClpI03)FWcx(m5(P1DW+2-{Hr-`5M{v##Zu-i-9Cvt;V|n)1pR^y ztp3IXzHjYWqabuPqnCY9^^;adc!a%Z35VN~TzwAxq{NU&Kp35m?fw_^D{wzB}4FVXX5Zk@#={6jRh%wx|!eu@Xp;%x+{2;}!&J4X*_SvtkqE#KDIPPn@ z5BE$3uRlb>N<2A$g_cuRQM1T#5ra9u2x9pQuqF1l2#N{Q!jVJ<>HlLeVW|fN|#vqSnRr<0 zTVs=)7d`=EsJXkZLJgv~9JB&ay16xDG6v(J2eZy;U%a@EbAB-=C?PpA9@}?_Yfb&) zBpsih5m1U9Px<+2$TBJ@7s9HW>W){i&XKLZ_{1Wzh-o!l5_S+f$j^RNYo85}uVhN# zq}_mN-d=n{>fZD2Lx$Twd2)}X2ceasu91}n&BS+4U9=Y{aZCgV5# z?z_Hq-knIbgIpnkGzJz-NW*=p?3l(}y3(aPCW=A({g9CpjJfYuZ%#Tz81Y)al?!S~ z9AS5#&nzm*NF?2tCR#|D-EjBWifFR=da6hW^PHTl&km-WI9*F4o>5J{LBSieVk`KO z2(^9R(zC$@g|i3}`mK-qFZ33PD34jd_qOAFj29687wCUy>;(Hwo%Me&c=~)V$ua)V zsaM(aThQ3{TiM~;gTckp)LFvN?%TlO-;$y+YX4i`SU0hbm<})t0zZ!t1=wY&j#N>q zONEHIB^RW6D5N*cq6^+?T}$3m|L{Fe+L!rxJ=KRjlJS~|z-&CC{#CU8`}2|lo~)<| zk?Wi1;Cr;`?02-C_3^gD{|Ryhw!8i?yx5i0v5?p)9wZxSkwn z3C;pz25KR&7{|rc4H)V~y8%+6lX&KN&=^$Wqu+}}n{Y~K4XpI-#O?L=(2qncYNePX zTsB6_3`7q&e0K67=Kg7G=j#?r!j0S^w7;0?CJbB3_C4_8X*Q%F1%cmB{g%XE&|IA7 z(#?AeG{l)s_orNJp!$Q~qGrj*YnuKlV`nVdg4vkTNS~w$4d^Oc3(dxi(W5jq0e>x} z(GN1?u2%Sy;GA|B%Sk)ukr#v*UJU%(BE9X54!&KL9A^&rR%v zIdYt0&D59ggM}CKWyxGS@ z>T#})2Bk8sZMGJYFJtc>D#k0+Rrrs)2DG;(u(DB_v-sVg=GFMlSCx<&RL;BH}d6AG3VqP!JpC0Gv6f8d|+7YRC@g|=N=C2 zo>^0CE0*RW?W))S(N)}NKA)aSwsR{1*rs$(cZIs?nF9)G*bSr%%SZo^YQ|TSz={jX z4Z+(~v_>RH0(|IZ-_D_h@~p_i%k^XEi+CJVC~B zsPir zA0Jm2yIdo4`&I`hd%$Bv=Rq#-#bh{Mxb_{PN%trcf(#J3S1UKDfC1QjH2E;>wUf5= ze8tY9QSYx0J;$JUR-0ar6fuiQTCQP#P|WEq;Ez|*@d?JHu-(?*tTpGHC+=Q%H>&I> z*jC7%nJIy+HeoURWN%3X47UUusY2h7nckRxh8-)J61Zvn@j-uPA@99|y48pO)0XcW zX^d&kW^p7xsvdX?2QZ8cEUbMZ7`&n{%Bo*xgFr4&fd#tHOEboQos~xm8q&W;fqrj} z%KYnnE%R`=`+?lu-O+J9r@+$%YnqYq!SVs>xp;%Q8p^$wA~oynhnvIFp^)Z2CvcyC zIN-_3EUHW}1^VQ0;Oj>q?mkPx$Wj-i7QoXgQ!HyRh6Gj8p~gH22k&nmEqUR^)9qni{%uNeV{&0-H60C zibHZtbV=8=aX!xFvkO}T@lJ_4&ki$d+0ns3FXb+iP-VAVN`B7f-hO)jyh#4#_$XG%Txk6M<+q6D~ zi*UcgRBOoP$7P6RmaPZ2%MG}CMfs=>*~(b97V4+2qdwvwA@>U3QQAA$hiN9zi%Mq{ z*#fH57zUmi)GEefh7@`Uy7?@@=BL7cXbd{O9)*lJh*v!@ z-6}p9u0AreiGauxn7JBEa-2w&d=!*TLJ49`U@D7%2ppIh)ynMaAE2Q4dl@47cNu{9 z&3vT#pG$#%hrXzXsj=&Ss*0;W`Jo^mcy4*L8b^sSi;H{*`zW9xX2HAtQ*sO|x$c6UbRA(7*9=;D~(%wfo(Z6#s$S zuFk`dr%DfVX5KC|Af8@AIr8@OAVj=6iX!~8D_P>p7>s!Hj+X0_t}Y*T4L5V->A@Zx zcm1wN;TNq=h`5W&>z5cNA99U1lY6+!!u$ib|41VMcJk8`+kP{PEOUvc@2@fW(bh5pp6>C3T55@XlpsAd#vn~__3H;Dz2w=t9v&{v*)1m4)vX;4 zX4YAjM66?Z7kD@XX{e`f1t_ZvYyi*puSNhVPq%jeyBteaOHo7vOr8!qqp7wV;)%jtD5>}-a?xavZ;i|2P3~7c)vP2O#Fb`Y&Kce zQNr7%fr4#S)OOV-1piOf7NgQvR{lcvZ*SNbLMq(olrdDC6su;ubp5un!&oT=jVTC3uTw7|r;@&y*s)a<{J zkzG(PApmMCpMmuh6GkM_`AsBE@t~)EDcq1AJ~N@7bqyW_i!mtHGnVgBA`Dxi^P93i z5R;}AQ60wy=Q2GUnSwz+W6C^}qn`S-lY7=J(3#BlOK%pCl=|RVWhC|IDj1E#+|M{TV0vE;vMZLy7KpD1$Yk zi0!9%qy8>CyrcRK`juQ)I};r)5|_<<9x)32b3DT1M`>v^ld!yabX6@ihf`3ZVTgME zfy(l-ocFuZ(L&OM4=1N#Mrrm_<>1DZpoWTO70U8+x4r3BpqH6z@(4~sqv!A9_L}@7 z7o~;|?~s-b?ud&Wx6==9{4uTcS|0-p@dKi0y#tPm2`A!^o3fZ8Uidxq|uz2vxf;wr zM^%#9)h^R&T;}cxVI(XX7kKPEVb);AQO?cFT-ub=%lZPwxefymBk+!H!W(o(>I{jW z$h;xuNUr#^0ivvSB-YEbUqe$GLSGrU$B3q28&oA55l)ChKOrwiTyI~e*uN;^V@g-Dm4d|MK!ol8hoaSB%iOQ#i_@`EYK_9ZEjFZ8Ho7P^er z^2U6ZNQ{*hcEm?R-lK)pD_r(e=Jfe?5VkJ$2~Oq^7YjE^5(6a6Il--j@6dBHx2Ulq z!%hz{d-S~i9Eo~WvQYDt7O7*G9CP#nrKE#DtIEbe_uxptcCSmYZMqT2F}7Kw0AWWC zPjwo0IYZ6klc(h9uL|NY$;{SGm4R8Bt^^q{e#foMxfCSY^-c&IVPl|A_ru!ebwR#7 z3<4+nZL(mEsU}O9e`^XB4^*m)73hd04HH%6ok^!;4|JAENnEr~%s6W~8KWD)3MD*+ zRc46yo<}8|!|yW-+KulE86aB_T4pDgL$XyiRW(OOcnP4|2;v!m2fB7Hw-IkY#wYfF zP4w;k-RInWr4fbz=X$J;z2E8pvAuy9kLJUSl8_USi;rW`kZGF?*Ur%%(t$^{Rg!=v zg;h3@!Q$eTa7S0#APEDHLvK%RCn^o0u!xC1Y0Jg!Baht*a4mmKHy~88md{YmN#x) zBOAp_i-z2h#V~*oO-9k(BizR^l#Vm%uSa^~3337d;f=AhVp?heJ)nlZGm`}D(U^2w z#vC}o1g1h?RAV^90N|Jd@M00PoNUPyA?@HeX0P7`TKSA=*4s@R;Ulo4Ih{W^CD{c8 ze(ipN{CAXP(KHJ7UvpOc@9SUAS^wKo3h-}BDZu}-qjdNlVtp^Z{|CxKOEo?tB}-4; zEXyDzGbXttJ3V$lLo-D?HYwZm7vvwdRo}P#KVF>F|M&eJ44n*ZO~0)#0e0Vy&j00I z{%IrnUvKp70P?>~J^$^0Wo%>le>re2ZSvRfes@dC-*e=DD1-j%<$^~4^4>Id5w^Fr z{RWL>EbUCcyC%1980kOYqZAcgdz5cS8c^7%vvrc@CSPIx;X=RuodO2dxk17|am?HJ@d~Mp_l8H?T;5l0&WGFoTKM{eP!L-a0O8?w zgBPhY78tqf^+xv4#OK2I#0L-cSbEUWH2z+sDur85*!hjEhFfD!i0Eyr-RRLFEm5(n z-RV6Zf_qMxN5S6#8fr9vDL01PxzHr7wgOn%0Htmvk9*gP^Um=n^+7GLs#GmU&a#U^4jr)BkIubQO7oUG!4CneO2Ixa`e~+Jp9m{l6apL8SOqA^ zvrfEUPwnHQ8;yBt!&(hAwASmL?Axitiqvx%KZRRP?tj2521wyxN3ZD9buj4e;2y6U zw=TKh$4%tt(eh|y#*{flUJ5t4VyP*@3af`hyY^YU3LCE3Z|22iRK7M7E;1SZVHbXF zKVw!L?2bS|kl7rN4(*4h2qxyLjWG0vR@`M~QFPsf^KParmCX;Gh4OX6Uy9#4e_%oK zv1DRnfvd$pu(kUoV(MmAc09ckDiuqS$a%!AQ1Z>@DM#}-yAP$l`oV`BDYpkqpk(I|+qk!yoo$TwWr6dRzLy(c zi+qbVlYGz0XUq@;Fm3r~_p%by)S&SVWS+wS0rC9bk^3K^_@6N5|2rtF)wI>WJ=;Fz zn8$h<|Dr%kN|nciMwJAv;_%3XG9sDnO@i&pKVNEfziH_gxKy{l zo`2m4rnUT(qenuq9B0<#Iy(RPxP8R)=5~9wBku=%&EBoZ82x1GlV<>R=hIqf0PK!V zw?{z9e^B`bGyg2nH!^x}06oE%J_JLk)^QyHLipoCs2MWIqc>vaxsJj(=gg1ZSa=u{ zt}od#V;e7sA4S(V9^<^TZ#InyVBFT(V#$fvI7Q+pgsr_2X`N~8)IOZtX}e(Bn(;eF zsNj#qOF_bHl$nw5!ULY{lNx@93Fj}%R@lewUuJ*X*1$K`DNAFpE z7_lPE+!}uZ6c?+6NY1!QREg#iFy=Z!OEW}CXBd~wW|r_9%zkUPR0A3m+@Nk%4p>)F zXVut7$aOZ6`w}%+WV$te6-IX7g2yms@aLygaTlIv3=Jl#Nr}nN zp|vH-3L03#%-1-!mY`1z?+K1E>8K09G~JcxfS)%DZbteGQnQhaCGE2Y<{ut#(k-DL zh&5PLpi9x3$HM82dS!M?(Z zEsqW?dx-K_GMQu5K54pYJD=5+Rn&@bGjB?3$xgYl-|`FElp}?zP&RAd<522c$Rv6} zcM%rYClU%JB#GuS>FNb{P2q*oHy}UcQ-pZ2UlT~zXt5*k-ZalE(`p7<`0n7i(r2k{ zb84&^LA7+aW1Gx5!wK!xTbw0slM?6-i32CaOcLC2B>ZRI16d{&-$QBEu1fKF0dVU>GTP05x2>Tmdy`75Qx! z^IG;HB9V1-D5&&)zjJ&~G}VU1-x7EUlT3QgNT<&eIDUPYey$M|RD6%mVkoDe|;2`8Z+_{0&scCq>Mh3hj|E*|W3;y@{$qhu77D)QJ` znD9C1AHCKSAHQqdWBiP`-cAjq7`V%~JFES1=i-s5h6xVT<50kiAH_dn0KQB4t*=ua zz}F@mcKjhB;^7ka@WbSJFZRPeYI&JFkpJ-!B z!ju#!6IzJ;D@$Qhvz9IGY5!%TD&(db3<*sCpZ?U#1^9RWQ zs*O-)j!E85SMKtoZzE^8{w%E0R0b2lwwSJ%@E}Lou)iLmPQyO=eirG8h#o&E4~eew z;h><=|4m0$`ANTOixHQOGpksXlF0yy17E&JksB4_(vKR5s$Ve+i;gco2}^RRJI+~R zWJ82WGigLIUwP!uSELh3AAs9HmY-kz=_EL-w|9}noKE#(a;QBpEx9 z4BT-zY=6dJT>72Hkz=9J1E=}*MC;zzzUWb@x(Ho8cU_aRZ?fxse5_Ru2YOvcr?kg&pt@v;{ai7G--k$LQtoYj+Wjk+nnZty;XzANsrhoH#7=xVqfPIW(p zX5{YF+5=k4_LBnhLUZxX*O?29olfPS?u*ybhM_y z*XHUqM6OLB#lyTB`v<BZ&YRs$N)S@5Kn_b3;gjz6>fh@^j%y2-ya({>Hd@kv{CZZ2e)tva7gxLLp z`HoGW);eRtov~Ro5tetU2y72~ zQh>D`@dt@s^csdfN-*U&o*)i3c4oBufCa0e|BwT2y%Y~=U7A^ny}tx zHwA>Wm|!SCko~UN?hporyQHRUWl3djIc722EKbTIXQ6>>iC!x+cq^sUxVSj~u)dsY zW8QgfZlE*2Os%=K;_vy3wx{0u!2%A)qEG-$R^`($%AOfnA^LpkB_}Dd7AymC)zSQr z>C&N8V57)aeX8ap!|7vWaK6=-3~ko9meugAlBKYGOjc#36+KJwQKRNa_`W@7;a>ot zdRiJkz?+QgC$b}-Owzuaw3zBVLEugOp6UeMHAKo2$m4w zpw?i%Lft^UtuLI}wd4(-9Z^*lVoa}11~+0|Hs6zAgJ01`dEA&^>Ai=mr0nC%eBd_B zzgv2G_~1c1wr*q@QqVW*Wi1zn=}KCtSwLjwT>ndXE_Xa22HHL_xCDhkM( zhbw+j4uZM|r&3h=Z#YrxGo}GX`)AZyv@7#7+nd-D?BZV>thtc|3jt30j$9{aIw9)v zDY)*fsSLPQTNa&>UL^RWH(vpNXT7HBv@9=*=(Q?3#H*crA2>KYx7Ab?-(HU~a275)MBp~`P)hhzSsbj|d`aBe(L*(;zif{iFJu**ZR zkL-tPyh!#*r-JVQJq>5b0?cCy!uSKef+R=$s3iA7*k*_l&*e!$F zYwGI;=S^0)b`mP8&Ry@{R(dPfykD&?H)na^ihVS7KXkxb36TbGm%X1!QSmbV9^#>A z-%X>wljnTMU0#d;tpw?O1W@{X-k*>aOImeG z#N^x?ehaaQd}ReQykp>i;92q@%$a!y1PNyPYDIvMm& zyYVwn;+0({W@3h(r&i#FuCDE)AC(y&Vu>4?1@j0|CWnhHUx4|zL7cdaA32RSk?wl% zMK^n42@i5AU>f70(huWfOwaucbaToxj%+)7hnG^CjH|O`A}+GHZyQ-X57(WuiyRXV zPf>0N3GJ<2Myg!sE4XJY?Z7@K3ZgHy8f7CS5ton0Eq)Cp`iLROAglnsiEXpnI+S8; zZn>g2VqLxi^p8#F#Laf3<00AcT}Qh&kQnd^28u!9l1m^`lfh9+5$VNv=?(~Gl2wAl zx(w$Z2!_oESg_3Kk0hUsBJ<;OTPyL(?z6xj6LG5|Ic4II*P+_=ac7KRJZ`(k2R$L# zv|oWM@116K7r3^EL*j2ktjEEOY9c!IhnyqD&oy7+645^+@z5Y|;0+dyR2X6^%7GD* zXrbPqTO}O={ z4cGaI#DdpP;5u?lcNb($V`l>H7k7otl_jQFu1hh>=(?CTPN#IPO%O_rlVX}_Nq;L< z@YNiY>-W~&E@=EC5%o_z<^3YEw)i_c|NXxHF{=7U7Ev&C`c^0Z4-LGKXu*Hkk&Av= zG&RAv{cR7o4${k~f{F~J48Ks&o(D@j-PQ2`LL@I~b=ifx3q!p6`d>~Y!<-^mMk3)e zhi1;(YLU5KH}zzZNhl^`0HT(r`5FfmDEzxa zk&J7WQ|!v~TyDWdXQ)!AN_Y%xM*!jv^`s)A`|F%;eGg27KYsrCE2H}7*r)zvum6B{ z$k5Har9pv!dcG%f|3hE(#hFH+12RZPycVi?2y`-9I7JHryMn3 z9Y8?==_(vOAJ7PnT<0&85`_jMD0#ipta~Q3M!q5H1D@Nj-YXI$W%OQplM(GWZ5Lpq z-He6ul|3<;ZQsqs!{Y7x`FV@pOQc4|N;)qgtRe(Uf?|YqZv^$k8On7DJ5>f2%M=TV zw~x}9o=mh$JVF{v4H5Su1pq66+mhTG6?F>Do}x{V(TgFwuLfvNP^ijkrp5#s4UT!~ zEU7pr8aA)2z1zb|X9IpmJykQcqI#(rS|A4&=TtWu@g^;JCN`2kL}%+K!KlgC z>P)v+uCeI{1KZpewf>C=?N7%1e10Y3pQCZST1GT5fVyB1`q)JqCLXM zSN0qlreH1=%Zg-5`(dlfSHI&2?^SQdbEE&W4#%Eve2-EnX>NfboD<2l((>>34lE%) zS6PWibEvuBG7)KQo_`?KHSPk+2P;`}#xEs}0!;yPaTrR#j(2H|#-CbVnTt_?9aG`o z(4IPU*n>`cw2V~HM#O`Z^bv|cK|K};buJ|#{reT8R)f+P2<3$0YGh!lqx3&a_wi2Q zN^U|U$w4NP!Z>5|O)>$GjS5wqL3T8jTn%Vfg3_KnyUM{M`?bm)9oqZP&1w1)o=@+(5eUF@=P~ zk2B5AKxQ96n-6lyjh&xD!gHCzD$}OOdKQQk7LXS-fk2uy#h{ktqDo{o&>O!6%B|)` zg?|JgcH{P*5SoE3(}QyGc=@hqlB5w;bnmF#pL4iH`TSuft$dE5j^qP2S)?)@pjRQZ zBfo6g>c!|bN-Y|(Wah2o61Vd|OtXS?1`Fu&mFZ^yzUd4lgu7V|MRdGj3e#V`=mnk- zZ@LHn?@dDi=I^}R?}mZwduik!hC%=Hcl56u{Wrk1|1SxlgnzG&e7Vzh*wNM(6Y!~m z`cm8Ygc1$@z9u9=m5vs1(XXvH;q16fxyX4&e5dP-{!Kd555FD6G^sOXHyaCLka|8j zKKW^E>}>URx736WWNf?U6Dbd37Va3wQkiE;5F!quSnVKnmaIRl)b5rM_ICu4txs+w zj}nsd0I_VG^<%DMR8Zf}vh}kk;heOQTbl ziEoE;9@FBIfR7OO9y4Pwyz02OeA$n)mESpj zdd=xPwA`nO06uGGsXr4n>Cjot7m^~2X~V4yH&- zv2llS{|und45}Pm1-_W@)a-`vFBpD~>eVP(-rVHIIA|HD@%7>k8JPI-O*<7X{L*Ik zh^K`aEN!BteiRaY82FVo6<^8_22=aDIa8P&2A3V<(BQ;;x8Zs-1WuLRWjQvKv1rd2 zt%+fZ!L|ISVKT?$3iCK#7whp|1ivz1rV*R>yc5dS3kIKy_0`)n*%bfNyw%e7Uo}Mnnf>QwDgeH$X5eg_)!pI4EJjh6?kkG2oc6Af0py z(txE}$ukD|Zn=c+R`Oq;m~CSY{ebu9?!is}01sOK_mB?{lSY33E=!KkKtMeI*FO2b z%95awv9;Z|UDp3xm+aP*5I!R-_M2;GxeCRx3ATS0iF<_Do2Mi)Hk2 zjBF35VB>(oamIYjunu?g0O-?LuOvtfs5F(iiIicbu$HMPPF%F>pE@hIRjzT)>aa=m zwe;H9&+2|S!m74!E3xfO{l3E_ab`Q^tZ4yH9=~o2DUEtEMDqG=&D*8!>?2uao%w`&)THr z^>=L3HJquY>6)>dW4pCWbzrIB+>rdr{s}}cL_?#!sOPztRwPm1B=!jP7lQG|Iy6rP zVqZDNA;xaUx&xUt?Ox|;`9?oz`C0#}mc<1Urs#vTW4wd{1_r`eX=BeSV z_9WV*9mz>PH6b^z{VYQJ1nSTSqOFHE9u>cY)m`Q>=w1NzUShxcHsAxasnF2BG;NQ; zqL1tjLjImz_`q=|bAOr_i5_NEijqYZ^;d5y3ZFj6kCYakJh**N_wbfH;ICXq?-p#r z{{ljNDPSytOaG#7=yPmA&5gyYI%^7pLnMOw-RK}#*dk=@usL;|4US?{@K%7esmc&n z5$D*+l&C9)Bo@$d;Nwipd!68&+NnOj^<~vRcKLX>e03E|;to;$ndgR;9~&S-ly5gf z{rzj+j-g$;O|u?;wwxrEpD=8iFzUHQfl{B>bLHqH(9P zI59SS2PEBE;{zJUlcmf(T4DrcO?XRWR}?fekN<($1&AJTRDyW+D*2(Gyi?Qx-i}gy z&BpIO!NeVdLReO!YgdUfnT}7?5Z#~t5rMWqG+$N2n%5o#Np6ccNly}#IZQsW4?|NV zR9hrcyP(l#A+U4XcQvT;4{#i)dU>HK>aS!k1<3s2LyAhm2(!Nu%vRC9T`_yn9D+r} z1i&U~IcQ?4xhZYyH6WL-f%}qIhZkc&}n2N0PM| z6|XA9d-y;!`D{p;xu*gv7a|zaZ*MiQ)}zPzW4GB0mr)}N-DmB&hl1&x`2@sxN572_ zS)RdJyR%<7kW0v3Q_|57JKy&9tUdbqz}|hwn84}U*0r^jt6Ssrp+#1y=JBcZ+F`f(N?O0XL1OFGN`1-r?S<#t4*C9|y~e)!UYZ zRQ3M8m%~M)VriIvn~XzoP;5qeu(ZI>Y#r zAd)J)G9)*BeE%gmm&M@Olg3DI_zokjh9NvdGbT z+u4(Y&uC6tBBefIg~e=J#8i1Zxr>RT)#rGaB2C71usdsT=}mm`<#WY^6V{L*J6v&l z1^Tkr6-+^PA)yC;s1O^3Q!)Reb=fxs)P~I*?i&j{Vbb(Juc?La;cA5(H7#FKIj0Or zgV0BO{DUs`I9HgQ{-!g@5P^Vr|C4}~w6b=#`Zx0XcVSd?(04HUHwK(gJNafgQNB9Z zCi3TgNXAeJ+x|X|b@27$RxuYYuNSUBqo#uyiH6H(b~K*#!@g__4i%HP5wb<+Q7GSb zTZjJw96htUaGZ89$K_iBo4xEOJ#DT#KRu9ozu!GH0cqR>hP$nk=KXM%Y!(%vWQ#}s zy=O#BZ>xjUejMH^F39Bf0}>D}yiAh^toa-ts#gt6Mk9h1D<9_mGMBhLT0Ce2O3d_U znaTkBaxd-8XgwSp5)x-pqX5=+{cSuk6kyl@k|5DQ!5zLUVV%1X9vjY0gerbuG6nwZu5KDMdq(&UMLZ zy?jW#F6joUtVyz`Y?-#Yc0=i*htOFwQ3`hk$8oq35D}0m$FAOp#UFTV3|U3F>@N?d zeXLZCZjRC($%?dz(41e~)CN10qjh^1CdAcY(<=GMGk@`b1ptA&L*{L@_M{%Vd5b*x#b1(qh=7((<_l%ZUaHtmgq} zjchBdiis{Afxf@3CjPR09E*2#X(`W#-n`~6PcbaL_(^3tfDLk?Nb6CkW9v!v#&pWJ3iV-9hz zngp#Q`w`r~2wt&cQ9#S7z0CA^>Mzm7fpt72g<0y-KT{G~l-@L#edmjZQ}7{*$mLgSdJfS$Ge{hrD=mr;GD)uYq8}xS zT>(w_;}894Kb}(P5~FOpFIEjadhmxD(PsZbKwa-qxVa7Oc7~ebPKMeN(pCRzq8s@l z`|l^*X1eK1+Spz--WkSW_nK`Cs@JmkY4+p=U91nJoy{tSH;TzuIyS)Q_(S@;Iakua zpuDo5W54Mo;jY@Ly1dY)j|+M%$FJ0`C=FW#%UvOd&?p}0QqL20Xt!#pr8ujy6CA-2 zFz6Ex5H1i)c9&HUNwG{8K%FRK7HL$RJwvGakleLLo}tsb>t_nBCIuABNo$G--_j!gV&t8L^4N6wC|aLC)l&w04CD6Vc#h^(YH@Zs4nwUGkhc_-yt{dK zMZ<%$swLmUl8`E~RLihGt@J5v;r;vT&*Q!Cx zZ55-zpb;W7_Q{tf$mQvF61(K>kwTq0x{#Din||)B{+6O#ArLi)kiHWVC4`fOT&B(h zw&YV`J1|^FLx~9Q%r-SFhYl4PywI7sF2Q$>4o50~dfp5nn}XHv-_DM?RGs#+4gM;% znU>k=81G~f6u%^Z{bcX&sUv*h|L+|mNq=W43y@{~C zpL-TW3hYPs0^*OqS#KQwA^CGG_A-6#`_{1LBCD&*3nY0UHWJj1D|VP%oQlFxLllaA zVI@2^)HZ%E*=RbQcFOKIP7?+|_xVK+2oG(t_EGl2y;Ovox zZb^qVpe!4^reKvpIBFzx;Ji=PmrV>uu-Hb>`s?k?YZQ?>av45>i(w0V!|n?AP|v5H zm`e&Tgli#lqGEt?=(?~fy<(%#nDU`O@}Vjib6^rfE2xn;qgU6{u36j_+Km%v*2RLnGpsvS+THbZ>p(B zgb{QvqE?~50pkLP^0(`~K& zjT=2Pt2nSnwmnDFi2>;*C|OM1dY|CAZ5R|%SAuU|5KkjRM!LW_)LC*A zf{f>XaD+;rl6Y>Umr>M8y>lF+=nSxZX_-Z7lkTXyuZ(O6?UHw^q; z&$Zsm4U~}KLWz8>_{p*WQ!OgxT1JC&B&>|+LE3Z2mFNTUho<0u?@r^d=2 z-av!n8r#5M|F%l;=D=S1mGLjgFsiYAOODAR}#e^a8 zfVt$k=_o}kt3PTz?EpLkt54dY}kyd$rU zVqc9SN>0c z753j-gdN~UiW*FUDMOpYEkVzP)}{Ds*3_)ZBi)4v26MQr140|QRqhFoP=a|;C{#KS zD^9b-9HM11W+cb1Y)HAuk<^GUUo(ut!5kILBzAe)Vaxwu4Up!7Ql*#DDu z>EB84&xSrh>0jT!*X81jJQq$CRHqNj29!V3FN9DCx)~bvZbLwSlo3l^zPb1sqBnp) zfZpo|amY^H*I==3#8D%x3>zh#_SBf?r2QrD(Y@El!wa;Ja6G9Y1947P*DC|{9~nO& z*vDnnU!8(cV%HevsraF%Y%2{Z>CL0?64eu9r^t#WjW4~3uw8d}WHzsV%oq-T)Y z0-c!FWX5j1{1##?{aTeCW2b$PEnwe;t`VPCm@sQ`+$$L2=3kBR%2XU1{_|__XJ$xt zibjY2QlDVs)RgHH*kl&+jn*JqquF)k_Ypibo00lcc<2RYqsi-G%}k0r(N97H7JEn7@E3ZTH0JK>d8)E~A-D z!B&z9zJw0Bi^fgQZI%LirYaBKnWBXgc`An*qvO^*$xymqKOp(+3}IsnVhu?YnN7qz zNJxDN-JWd7-vIiv2M9ih>x3gNVY%DzzY~dCnA}76IRl!`VM=6=TYQ=o&uuE8kHqZT zoUNod0v+s9D)7aLJ|hVqL0li1hg)%&MAciI(4YJ=%D4H$fGQ&Lu-?@>>@pEgC;ERrL= zI^cS&3q8fvEGTJZgZwL5j&jp%j9U^Of6pR{wA^u=tVt#yCQepXNIbynGnuWbsC_EE zRyMFq{5DK692-*kyGy~An>AdVR9u___fzmmJ4;^s0yAGgO^h{YFmqJ%ZJ_^0BgCET zE6(B*SzeZ4pAxear^B-YW<%BK->X&Cr`g9_;qH~pCle# zdY|UB5cS<}DFRMO;&czbmV(?vzikf)Ks`d$LL801@HTP5@r><}$xp}+Ip`u_AZ~!K zT}{+R9Wkj}DtC=4QIqJok5(~0Ll&_6PPVQ`hZ+2iX1H{YjI8axG_Bw#QJy`6T>1Nn z%u^l`>XJ{^vX`L0 z1%w-ie!dE|!SP<>#c%ma9)8K4gm=!inHn2U+GR+~ zqZVoa!#aS0SP(|**WfQSe?cA=1|Jwk`UDsny%_y{@AV??N>xWekf>_IZLUEK3{Ksi zWWW$if&Go~@Oz)`#=6t_bNtD$d9FMBN#&97+XKa+K2C@I9xWgTE{?Xnhc9_KKPcujj@NprM@e|KtV_SR+ zSpeJ!1FGJ=Te6={;;+;a46-*DW*FjTnBfeuzI_=I1yk8M(}IwEIGWV0Y~wia;}^dg z{BK#G7^J`SE10z4(_Me=kF&4ld*}wpNs91%2Ute>Om`byv9qgK4VfwPj$`axsiZ)wxS4k4KTLb-d~!7I@^Jq`>?TrixHk|9 zqCX7@sWcVfNP8N;(T>>PJgsklQ#GF>F;fz_Rogh3r!dy*0qMr#>hvSua;$d z3TCZ4tlkyWPTD<=5&*bUck~J;oaIzSQ0E03_2x{?weax^jL3o`ZP#uvK{Z5^%H4b6 z%Kbp6K?>{;8>BnQy64Jy$~DN?l(ufkcs6TpaO&i~dC>0fvi-I^7YT#h?m;TVG|nba%CKRG%}3P*wejg) zI(ow&(5X3HR_xk{jrnkA-hbwxEQh|$CET9Qv6UpM+-bY?E!XVorBvHoU59;q<9$hK z%w5K-SK zWT#1OX__$ceoq0cRt>9|)v}$7{PlfwN}%Wh3rwSl;%JD|k~@IBMd5}JD#TOvp=S57 zae=J#0%+oH`-Av}a(Jqhd4h5~eG5ASOD)DfuqujI6p!;xF_GFcc;hZ9k^a7c%%h(J zhY;n&SyJWxju<+r`;pmAAWJmHDs{)V-x7(0-;E?I9FWK@Z6G+?7Py8uLc2~Fh1^0K zzC*V#P88(6U$XBjLmnahi2C!a+|4a)5Ho5>owQw$jaBm<)H2fR=-B*AI8G@@P-8I8 zHios92Q6Nk-n0;;c|WV$Q);Hu4;+y%C@3alP`cJ2{z~*m-@de%OKVgiWp;4Q)qf9n zJ!vmx(C=_>{+??w{U^Bh|LFJ<6t}Er<-Tu{C{dv8eb(kVQ4!fOuopTo!^x1OrG}0D zR{A#SrmN`=7T29bzQ}bwX8OUufW9d9T4>WY2n15=k3_rfGOp6sK0oj7(0xGaEe+-C zVuWa;hS*MB{^$=0`bWF(h|{}?53{5Wf!1M%YxVw}io4u-G2AYN|FdmhI13HvnoK zNS2fStm=?8ZpKt}v1@Dmz0FD(9pu}N@aDG3BY8y`O*xFsSz9f+Y({hFx;P_h>ER_& z`~{z?_vCNS>agYZI?ry*V96_uh;|EFc0*-x*`$f4A$*==p`TUVG;YDO+I4{gJGrj^ zn?ud(B4BlQr;NN?vaz_7{&(D9mfd z8esj=a4tR-ybJjCMtqV8>zn`r{0g$hwoWRUI3}X5=dofN){;vNoftEwX>2t@nUJro z#%7rpie2eH1sRa9i6TbBA4hLE8SBK@blOs=ouBvk{zFCYn4xY;v3QSM%y6?_+FGDn z4A;m)W?JL!gw^*tRx$gqmBXk&VU=Nh$gYp+Swu!h!+e(26(6*3Q!(!MsrMiLri`S= zKItik^R9g!0q7y$lh+L4zBc-?Fsm8`CX1+f>4GK7^X2#*H|oK}reQnT{Mm|0ar<+S zRc_dM%M?a3bC2ILD`|;6vKA`a3*N~(cjw~Xy`zhuY2s{(7KLB{S>QtR3NBQ3>vd+= z#}Q)AJr7Y_-eV(sMN#x!uGX08oE*g=grB*|bBs}%^3!RVA4f%m3=1f0K=T^}iI&2K zuM2GG5_%+#v-&V>?x4W9wQ|jE2Q7Be8mOyJtZrqn#gXy-1fF1P$C8+We&B*-pi#q5 zETp%H6g+%#sH+L4=ww?-h;MRCd2J9zwQUe4gHAbCbH08gDJY;F6F)HtWCRW1fLR;)ysGZanlz*a+|V&@(ipWdB!tz=m_0 z6F}`d$r%33bw?G*azn*}Z;UMr{z4d9j~s`0*foZkUPwpJsGgoR0aF>&@DC;$A&(av z?b|oo;`_jd>_5nye`DVOcMLr-*Nw&nA z82E8Dw^$Lpso)gEMh?N|Uc^X*NIhg=U%enuzZOGi-xcZRUZmkmq~(cP{S|*+A6P;Q zprIkJkIl51@ng)8cR6QSXJtoa$AzT@*(zN3M+6`BTO~ZMo0`9$s;pg0HE3C;&;D@q zd^0zcpT+jC%&=cYJF+j&uzX87d(gP9&kB9|-zN=69ymQS9_K@h3ph&wD5_!4q@qI@ zBMbd`2JJ2%yNX?`3(u&+nUUJLZ=|{t7^Rpw#v-pqD2_3}UEz!QazhRty%|Q~WCo7$ z+sIugHA%Lmm{lBP#bnu_>G}Ja<*6YOvSC;89z67M%iG0dagOt1HDpDn$<&H0DWxMU zxOYaaks6%R@{`l~zlZ*~2}n53mn2|O&gE+j*^ypbrtBv{xd~G(NF?Z%F3>S6+qcry z?ZdF9R*a;3lqX_!rI(Cov8ER_mOqSn6g&ZU(I|DHo7Jj`GJ}mF;T(vax`2+B8)H_D zD0I;%I?*oGD616DsC#j0x*p+ZpBfd=9gR|TvB)832CRhsW_7g&WI@zp@r7dhg}{+4f=(cO2s+)jg0x(*6|^+6W_=YIfSH0lTcK* z%)LyaOL6em@*-_u)}Swe8rU)~#zT-vNiW(D*~?Zp3NWl1y#fo!3sK-5Ek6F$F5l3| zrFFD~WHz1}WHmzzZ!n&O8rTgfytJG*7iE~0`0;HGXgWTgx@2fD`oodipOM*MOWN-} zJY-^>VMEi8v23ZlOn0NXp{7!QV3F1FY_URZjRKMcY(2PV_ms}EIC^x z=EYB5UUQ{@R~$2Mwiw$_JAcF+szKB*n(`MYpDCl>~ss54uDQ%Xf-8|dgO zY)B_qju=IaShS|XsQo=nSYxV$_vQR@hd~;qW)TEfU|BA0&-JSwO}-a*T;^}l;MgLM zz}CjPlJX|W2vCzm3oHw3vqsRc3RY=2()}iw_k2#eKf&VEP7TQ;(DDzEAUgj!z_h2Br;Z3u=K~LqM6YOrlh)v9`!n|6M-s z?XvA~y<5?WJ{+yM~uPh7uVM&g-(;IC3>uA}ud?B3F zelSyc)Nx>(?F=H88O&_70%{ATsLVTAp88F-`+|egQ7C4rpIgOf;1tU1au+D3 zlz?k$jJtTOrl&B2%}D}8d=+$NINOZjY$lb{O<;oT<zXoAp01KYG$Y4*=)!&4g|FL(!54OhR-?)DXC&VS5E|1HGk8LY;)FRJqnz zb_rV2F7=BGwHgDK&4J3{%&IK~rQx<&Kea|qEre;%A~5YD6x`mo>mdR)l?Nd%T2(5U z_ciT02-zt_*C|vn?BYDuqSFrk3R(4B0M@CRFmG{5sovIq4%8AhjXA5UwRGo)MxZlI zI%vz`v8B+#ff*XtGnciczFG}l(I}{YuCco#2E6|+5WJ|>BSDfz0oT+F z%QI^ixD|^(AN`MS6J$ zXlKNTFhb>KDkJp*4*LaZ2WWA5YR~{`={F^hwXGG*rJYQA7kx|nwnC58!eogSIvy{F zm1C#9@$LhK^Tl>&iM0wsnbG7Y^MnQ=q))MgApj4)DQt!Q5S`h+5a%c7M!m%)?+h65 z0NHDiEM^`W+M4)=q^#sk(g!GTpB}edwIe>FJQ+jAbCo#b zXmtd3raGJNH8vnqMtjem<_)9`gU_-RF&ZK!aIenv7B2Y0rZhon=2yh&VsHzM|`y|0x$Zez$bUg5Nqj?@~^ zPN43MB}q0kF&^=#3C;2T*bDBTyO(+#nZnULkVy0JcGJ36or7yl1wt7HI_>V7>mdud zv2II9P61FyEXZuF$=69dn%Z6F;SOwyGL4D5mKfW)q4l$8yUhv7|>>h_-4T*_CwAyu7;DW}_H zo>N_7Gm6eed=UaiEp_7aZko@CC61@(E1be&5I9TUq%AOJW>s^9w%pR5g2{7HW9qyF zh+ZvX;5}PN0!B4q2FUy+C#w5J?0Tkd&S#~94(AP4%fRb^742pgH7Tb1))siXWXHUT z1Wn5CG&!mGtr#jq6(P#!ck@K+FNprcWP?^wA2>mHA03W?kj>5b|P0ErXS) zg2qDTjQ|grCgYhrH-RapWCvMq5vCaF?{R%*mu}1)UDll~6;}3Q*^QOfj!dlt02lSzK z?+P)02Rrq``NbU3j&s*;<%i4Y>y9NK&=&KsYwvEmf5jwTG6?+Pu1q9M8lLlx)uZZ7 zizhr~e0ktGs-=$li-2jz^_48-jk**y&5u0`B2gc#i$T1~t+AS*kEfR*b{^Ec>2-F~ zKYRl&uQ5yO@EtAZX8ZSqx;8+AKf+CqhlUSpp*VfyBMv+%wxN5GukZEi^_to%MFRc0 zdXqJ*jk?#uYT6EJe446@(f6G4vhnxQP|pGeJ?-#|Ksq?g*ky=}x+Qnx+!<>Y(XStN zQIND`{KU}&l)E*ntI^}kJ=ly8DML{!(58Xk4_bzIc@v~e;>wKl_`7G%pGz~4KH*CTp;_|52)d!+ximd$|8v@zzEq%j68QXkgf$7eM~xdM5q5i z{?qFx_W|eq@L03bWJfjy^z@()-iCjzjREuf zb_a(yTz)ZKWCF%Lp>^2-%Q?*t{06}x#DLN3cO=i>h6#-a`z;<5rBGGM6GA(WqvRcX%Pn?Uvs1#e|ePSNJEC%+X(YI$x)`s$%>O#%}D9dgqWfq4yfVz^%FglokdFR}uJQhx|}_w`9Ulx38Ha>ZslKs58c-@IFI&f;?xM zbK>rKNfPFsf>%+k6%(A6=7Aac^_qrOCNqb3ZVJ;8pt!?1DR*ynJb#@II9h?)xB)A~ zm9Kk)Hy}!Z+W}i6ZJDy+?yY_=#kWrzgV)2eZAx_E=}Nh7*#<&mQz`Umfe$+l^P(xd zN}PA2qII4}ddCU+PN+yxkH%y!Qe(;iH3W%bwM3NKbU_saBo<8x9fGNtTAc_SizU=o zC3n2;c%LoU^j90Sz>B_p--Fzqv7x7*?|~-x{haH8RP)p|^u$}S9pD-}5;88pu0J~9 zj}EC`Q^Fw}`^pvAs4qOIuxKvGN@DUdRQ8p-RXh=3S#<`3{+Qv6&nEm)uV|kRVnu6f zco{(rJaWw(T0PWim?kkj9pJ)ZsUk9)dSNLDHf`y&@wbd;_ita>6RXFJ+8XC*-wsiN z(HR|9IF283fn=DI#3Ze&#y3yS5;!yoIBAH(v}3p5_Zr+F99*%+)cp!Sy8e+lG?dOc zuEz<;3X9Z5kkpL_ZYQa`sioR_@_cG z8tT~GOSTWnO~#?$u)AcaBSaV7P~RT?Nn8(OSL1RmzPWRWQ$K2`6*)+&7^zZBeWzud z*xb3|Fc~|R9eH+lQ#4wF#c;)Gka6lL(63C;>(bZob!i8F-3EhYU3|6-JBC0*5`y0| zBs!Frs=s!Sy0qmQNgIH|F`6(SrD1js2prni_QbG9Sv@^Pu2szR9NZl8GU89gWWvVg z2^-b*t+F{Nt>v?js7hnlC`tRU(an0qQG7;h6T~ z-`vf#R-AE$pzk`M{gCaia}F`->O2)60AuGFAJg> z*O2IZqTx=AzDvC49?A92>bQLdb&32_4>0Bgp0ESXXnd4B)!$t$g{*FG%HYdt3b3a^J9#so%BJMyr2 z{y?rzW!>lr097b9(75#&4&@lkB1vT*w&0E>!dS+a|ZOu6t^zro2tiP)bhcNNxn zbJs3_Fz+?t;4bkd8GfDI7ccJ5zU`Bs~ zN~bci`c`a%DoCMel<-KUCBdZRmew`MbZEPYE|R#|*hhvhyhOL#9Yt7$g_)!X?fK^F z8UDz)(zpsvriJ5aro5>qy`Fnz%;IR$@Kg3Z3EE!fv9CAdrAym6QU82=_$_N5*({_1 z7!-=zy(R{xg9S519S6W{HpJZ8Is|kQ!0?`!vxDggmslD59)>iQ15f z7J8NqdR`9f8H|~iFGNsPV!N)(CC9JRmzL9S}7U-K@`X893f3f<8|8Ls!^eA^#(O6nA+ByFIXcz_WLbfeG|nHJ5_sJJ^gNJ%SI9#XEfNRbzV+!RkI zXS$MOVYb2!0vU}Gt7oUy*|WpF^*orBot~b2J@^be?Gq;U%#am8`PmH-UCFZ&uTJlnetYij0z{K1mmivk$bdPbLodu;-R@@#gAV!=d%(caz$E?r zURX0pqAn7UuF6dULnoF1dZ$WM)tHAM{eZK6DbU1J`V5Dw<;xk}Nl`h+nfMO_Rdv z3SyOMzAbYaD;mkxA7_I_DOs#Bk;e5D%gsS3q)hlmi1w{FsjKNJE22`AjmNiAPRnIc zcIkN25;rOn3FipAFd(PnlK9{03w6Q<(68#1Jw`{axEGQE{Ac>^U$h);h2ADICmaNxrfpb`Jdr*)Y1SicpYKCFv$3vf~;5aW>n^7QGa63MJ z;B1+Z>WQ615R2D8JmmT`T{QcgZ+Kz1hTu{9FOL}Q8+iFx-Vyi}ZVVcGjTe>QfA`7W zFoS__+;E_rQIQxd(Bq4$egKeKsk#-9=&A!)(|hBvydsr5ts0Zjp*%*C0lM2sIOx1s zg$xz?Fh?x!P^!vWa|}^+SY8oZHub7f;E!S&Q;F?dZmvBxuFEISC}$^B_x*N-xRRJh zn4W*ThEWaPD*$KBr8_?}XRhHY7h^U1aN6>m=n~?YJQd8+!Uyq_3^)~4>XjelM&!c9 zCo|0KsGq7!KsZ~9@%G?i>LaU7#uSTMpypocm*oqJHR|wOgVWc7_8PVuuw>x{kEG4T z$p^DV`}jUK39zqFc(d5;N+M!Zd3zhZN&?Ww(<@AV-&f!v$uV>%z+dg9((35o@4rqLvTC-se@hkn^6k7+xHiK-vTRvM8{bCejbU;1@U=*r}GTI?Oc$!b6NRcj83-zF; z=TB#ESDB`F`jf4)z=OS76Se}tQDDHh{VKJk#Ad6FDB_=afpK#pyRkGrk~OuzmQG)} z*$t!nZu$KN&B;|O-aD=H<|n6aGGJZ=K9QFLG0y=Jye_ElJFNZJT;fU8P8CZcLBERjioAOC0Vz_pIXIc};)8HjfPwNy zE!g|lkRv3qpmU?shz(BBt5%TbpJC3HzP9!t7k*Fh48!-HlJ4TTgdCr3rCU!iF}kgu z4Qs;K@XOY~4f~N}Jl8V_mGbwzvNLbl&0e9UG4W;kvjTK|5`-Ld+eQ6YRF`N0ct%u% z^3J_{7r#_W1zm|>IPN!yWCRrN)N!7v`~ptNkIXKipQ6ogFvcnI5ugxdoa{d;uD67g zgo^}QuZRkB540Vc!@c80(wFG=$ct}oHq(#W0+-XX(;Rrt`x=<45X}ficNtI2(&}=~ zb(!}tNz?s`wm{gK?2tdf+OEF;tzx<(3fMd7_tM@Ghs$Z(Os-H(kYq#qB|J-aC9Ku?fsWwJhB36c)A zu|a7ZF?V8X7l2g5~xqZf>2=6Dsi5lfo zKIRL&@MLJyaBE)V_9=pJYu%U2wxR*-(0MI5_|yqP`?h@cks(5LR@XUKLMI_xuVtiu zRvpDS8MyUMRFM6`P+Sjc!A_e^H38Qu7b{b7QZ>NHyA6k-YYygQuW&C_OGO(7V7?}r)zedSVpBI zuk29Z4GW3C0GpfozbZQya454sjt@ndQmsp=DA&@sWw&xmOlDk1JIcMNp~-ES$&A~k zG#W(6hBj?!Fu8Q4WYexoSBa8_5=v20xnx6H?e;$t)5|f&{7=vOye^&3_c-Ug?|a@e z=X`&qT_5B7N9vZoPBhXOTEDV;4&x2Je4}T(UB~O-$D#CjX77$R?RZ*`ed~$G;$4YS z4n*|Pop(!NN79Hk2}U#cfEEwdxM)xQm}$~rV03xc=#U@@Y*}qEmot5KvDb=8{!E-n zl4p?}&g2h^sUGyTcGh=0aQzQb*k;K;dvbeZUgmwEv>%#(EPtj=gHKdi|E8@w+|>KC zxEU>b>P+9Xf}pEyQK(}#QrBG4Jaf!iE!qpMbTu>gb!gtdq<`@xO+roQl+S_7)!G(% zdy)$iGmJ1cwP?F=IyyV1-$|kf|EKM3B@I&lZ%NI@VV;*mQdLWjc#t|Vbk_Q~>&O03 zIcSr$(qLAINj7a z;!||v&1D5SX#X@5jNd}jUsi-CH_Scjyht&}q2p*CJCC-`&NyXf)vD5{e!HO629D-O z%bZelTcq=DoRX>zeWCa^RmR3*{x9;3lZ75M#S)!W0bRIFH#P6b%{|HRSZ5!!I#s)W z_|XXZQ<0_`>b^^0Z>LU64Yg1w)8}#M^9se(OZ9~baZ7fsKFc;EtnB>kesci#>=icG zuHdjax2^=!_(9?0l7;G7^-}9>Y#M zm;9*GT~dBuYWdk49%mZM0=H#FY1)}7NE5DE_vsqrA0`?0R0q535qHjWXcl|gz9Fq$ zMKxgL;68l!gm3y0durIr3LHv~y*ABm` zYhQG0UW#hg@*A{&G!;$FS43}rIF$e6yRdGJWVR<}uuJ_5_8qa3xaHH^!VzUteVp;> z<0`M>3tnY$ZFb$(`0sg93TwGyP;`9UYUWxO&CvAnSzei&ap))NcW;R`tA=y^?mBmG+M*&bqW5kL$V(O;(p)aEk`^ci?2Jwxu>0sy>a7+Wa9t z5#I2o;+gr^9^&km^z7>xJWbN&Ft>Vna34E zI@BBzwX)R}K3SL?)enrDJ45QLt;-7CFJk{`cF3L4Z^CtG_r5)0)HV>BOYPIUh#D%| zYQAu31f{bm-D*`_k7DTTr?Nkw_gY%J1cb2&TdtibY?V=|SSIOlA;|5C!2@?YQ z-$?G0jj^mG|MP>DmbF7}T~C$H6=CpZ~hd zZ1C|xV@=h#^~`3LSCnmI(vZ|5r3>eq5*UB)dhdy``*gKY3Eg%jSK8I-`G+OWWlD)T zt$wSQ=||lSkiKy}YF-k}@W9EiS?)z`hK{R!dd-$BCJvBtAN-yXn3njU$MisEtp!?Q z%Vk-*(wy9dd15(-WFw_&^tT;;IpF?ox1`Qq3-0zVTk+$W_?q}GfAQlPcrB^?&tWSI z2BB!K=sH7FUYmXa_dcV^Z3>5z8}~W{S!$jVR_3hu_|wl2|gmRH8ftn^z@fW75*;-`;wU+fY+BR_yx6BZnE5_Hna({jrPiubRp$jZ=T=t$hx&NeCV1!vuCcl4PJ0p0Fjp>6K} zHkoD1gQk=P2hYcT%)cJ2Q5WuA|5_x+dX0%hnozfTF>$#Wz~X!MY>){H4#fB#7^ID* z1*o2Hzp}?WVs&gbS?Uq(CT0sP+F)u9{xfgg6o_{8J#m;|NeJqDHhb(Q8%z8aM_qeM zn83>d`uDd47WIuKp78JBYo2SYupGcNXIzeou^eMY`@%Bv8elZ>q~3uq#~IX)g%g;h zoUXymEd>|kVsMkyb&1l~lrE-`w(0PObapYa35DJ4Y03Jv_!DKp}0HTbOgZRM=;PSsuAJJJ1 zItc+tu9;ANG;qHaCI|T85!euhFK~VK^G2LZV1+cbzS?>ar@>emg;JTI5VAn1g5U~| zU=p&k0OlSzc$U=s#9_uL3&n|6A1X$XvrE9vFV@`A4G#!D1QcFCeE`F2N(deJx>)*A z$XIW0P~-NbAd=5i6`s<~(vAQX9t$dbVqc5|E|CHRtb$1(l&KSNh_t2#k_l95KnP86 z)ns_DGspv-M0z0#h2a+*oH|{5~j{ zXGD=}cLrBSESQ0u$XmQlFfWMCAWaS;wKK%#aSSYK=qljBiY(s zT$v;We24&$w=avIILsMt0%1fDyah|AlLNg#WL$Lu)tf}YfqO%+pH~QC*bZO4aM*i9 zrPFf|5!hv@XY8CzaFh*Dy9vH|2fKKr(@x}`L#9^*vOae|lk`adG#oZZAyk|TOV8`9L zc-sQu%y1MQes&J?)a1}Zc*>-P!6j-T#75V$lLC!TuMB(!G-+D2;XptUxymSPFI-K&0x}B1?h$ z3-9**-9!);fwyiWB5gS$i;P~c=^}5-6G@{4TWDBRDc6(M|%qa-mS`z`u9kWo{Xl_uc;hXOkRd literal 0 HcmV?d00001 diff --git a/flow-jvm/gradle/wrapper/gradle-wrapper.properties b/flow-jvm/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000..17655d0ef2 --- /dev/null +++ b/flow-jvm/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/flow-jvm/gradlew b/flow-jvm/gradlew new file mode 100755 index 0000000000..fbd7c51583 --- /dev/null +++ b/flow-jvm/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/flow-jvm/gradlew.bat b/flow-jvm/gradlew.bat new file mode 100644 index 0000000000..a9f778a7a9 --- /dev/null +++ b/flow-jvm/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/flow-jvm/settings.gradle b/flow-jvm/settings.gradle new file mode 100644 index 0000000000..2969c7626f --- /dev/null +++ b/flow-jvm/settings.gradle @@ -0,0 +1,11 @@ +pluginManagement { + repositories { + mavenCentral() + gradlePluginPortal() + mavenLocal() + } + plugins { + id "${quarkusPluginId}" version "${quarkusPluginVersion}" + } +} +rootProject.name='flow-jvm' diff --git a/flow-jvm/src/main/docker/Dockerfile.jvm b/flow-jvm/src/main/docker/Dockerfile.jvm new file mode 100644 index 0000000000..37d52c9610 --- /dev/null +++ b/flow-jvm/src/main/docker/Dockerfile.jvm @@ -0,0 +1,97 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode +# +# Before building the container image run: +# +# ./gradlew build +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/flow-jvm-jvm . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/flow-jvm-jvm +# +# If you want to include the debug port into your docker image +# you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. +# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 +# when running the container +# +# Then run the container using : +# +# docker run -i --rm -p 8080:8080 quarkus/flow-jvm-jvm +# +# This image uses the `run-java.sh` script to run the application. +# This scripts computes the command line to execute your Java application, and +# includes memory/GC tuning. +# You can configure the behavior using the following environment properties: +# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class") +# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options +# in JAVA_OPTS (example: "-Dsome.property=foo") +# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is +# used to calculate a default maximal heap memory based on a containers restriction. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio +# of the container available memory as set here. The default is `50` which means 50% +# of the available memory is used as an upper boundary. You can skip this mechanism by +# setting this value to `0` in which case no `-Xmx` option is added. +# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This +# is used to calculate a default initial heap memory based on the maximum heap memory. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio +# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx` +# is used as the initial heap size. You can skip this mechanism by setting this value +# to `0` in which case no `-Xms` option is added (example: "25") +# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS. +# This is used to calculate the maximum value of the initial heap memory. If used in +# a container without any memory constraints for the container then this option has +# no effect. If there is a memory constraint then `-Xms` is limited to the value set +# here. The default is 4096MB which means the calculated value of `-Xms` never will +# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096") +# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output +# when things are happening. This option, if set to true, will set +# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true"). +# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example: +# true"). +# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787"). +# - CONTAINER_CORE_LIMIT: A calculated core limit as described in +# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2") +# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024"). +# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion. +# (example: "20") +# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking. +# (example: "40") +# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection. +# (example: "4") +# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus +# previous GC times. (example: "90") +# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20") +# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100") +# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should +# contain the necessary JRE command-line options to specify the required GC, which +# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC). +# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080") +# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080") +# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be +# accessed directly. (example: "foo.example.com,bar.example.com") +# +### +FROM registry.access.redhat.com/ubi8/openjdk-21:1.19 + +ENV LANGUAGE='en_US:en' + + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --chown=185 build/quarkus-app/lib/ /deployments/lib/ +COPY --chown=185 build/quarkus-app/*.jar /deployments/ +COPY --chown=185 build/quarkus-app/app/ /deployments/app/ +COPY --chown=185 build/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +USER 185 +ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + +ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ] + diff --git a/flow-jvm/src/main/docker/Dockerfile.legacy-jar b/flow-jvm/src/main/docker/Dockerfile.legacy-jar new file mode 100644 index 0000000000..439a10a1c2 --- /dev/null +++ b/flow-jvm/src/main/docker/Dockerfile.legacy-jar @@ -0,0 +1,93 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode +# +# Before building the container image run: +# +# ./gradlew build -Dquarkus.package.jar.type=legacy-jar +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.legacy-jar -t quarkus/flow-jvm-legacy-jar . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/flow-jvm-legacy-jar +# +# If you want to include the debug port into your docker image +# you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. +# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 +# when running the container +# +# Then run the container using : +# +# docker run -i --rm -p 8080:8080 quarkus/flow-jvm-legacy-jar +# +# This image uses the `run-java.sh` script to run the application. +# This scripts computes the command line to execute your Java application, and +# includes memory/GC tuning. +# You can configure the behavior using the following environment properties: +# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class") +# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options +# in JAVA_OPTS (example: "-Dsome.property=foo") +# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is +# used to calculate a default maximal heap memory based on a containers restriction. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio +# of the container available memory as set here. The default is `50` which means 50% +# of the available memory is used as an upper boundary. You can skip this mechanism by +# setting this value to `0` in which case no `-Xmx` option is added. +# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This +# is used to calculate a default initial heap memory based on the maximum heap memory. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio +# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx` +# is used as the initial heap size. You can skip this mechanism by setting this value +# to `0` in which case no `-Xms` option is added (example: "25") +# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS. +# This is used to calculate the maximum value of the initial heap memory. If used in +# a container without any memory constraints for the container then this option has +# no effect. If there is a memory constraint then `-Xms` is limited to the value set +# here. The default is 4096MB which means the calculated value of `-Xms` never will +# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096") +# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output +# when things are happening. This option, if set to true, will set +# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true"). +# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example: +# true"). +# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787"). +# - CONTAINER_CORE_LIMIT: A calculated core limit as described in +# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2") +# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024"). +# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion. +# (example: "20") +# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking. +# (example: "40") +# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection. +# (example: "4") +# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus +# previous GC times. (example: "90") +# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20") +# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100") +# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should +# contain the necessary JRE command-line options to specify the required GC, which +# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC). +# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080") +# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080") +# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be +# accessed directly. (example: "foo.example.com,bar.example.com") +# +### +FROM registry.access.redhat.com/ubi8/openjdk-21:1.19 + +ENV LANGUAGE='en_US:en' + + +COPY build/lib/* /deployments/lib/ +COPY build/*-runner.jar /deployments/quarkus-run.jar + +EXPOSE 8080 +USER 185 +ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + +ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ] diff --git a/flow-jvm/src/main/docker/Dockerfile.native b/flow-jvm/src/main/docker/Dockerfile.native new file mode 100644 index 0000000000..096c071eb0 --- /dev/null +++ b/flow-jvm/src/main/docker/Dockerfile.native @@ -0,0 +1,27 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. +# +# Before building the container image run: +# +# ./gradlew build -Dquarkus.native.enabled=true +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.native -t quarkus/flow-jvm . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/flow-jvm +# +### +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9 +WORKDIR /work/ +RUN chown 1001 /work \ + && chmod "g+rwX" /work \ + && chown 1001:root /work +COPY --chown=1001:root build/*-runner /work/application + +EXPOSE 8080 +USER 1001 + +ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"] diff --git a/flow-jvm/src/main/docker/Dockerfile.native-micro b/flow-jvm/src/main/docker/Dockerfile.native-micro new file mode 100644 index 0000000000..6842a77cfd --- /dev/null +++ b/flow-jvm/src/main/docker/Dockerfile.native-micro @@ -0,0 +1,30 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. +# It uses a micro base image, tuned for Quarkus native executables. +# It reduces the size of the resulting container image. +# Check https://quarkus.io/guides/quarkus-runtime-base-image for further information about this image. +# +# Before building the container image run: +# +# ./gradlew build -Dquarkus.native.enabled=true +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.native-micro -t quarkus/flow-jvm . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/flow-jvm +# +### +FROM quay.io/quarkus/quarkus-micro-image:2.0 +WORKDIR /work/ +RUN chown 1001 /work \ + && chmod "g+rwX" /work \ + && chown 1001:root /work +COPY --chown=1001:root build/*-runner /work/application + +EXPOSE 8080 +USER 1001 + +ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"] diff --git a/flow-jvm/src/main/java/io/peerdb/GreetingResource.java b/flow-jvm/src/main/java/io/peerdb/GreetingResource.java new file mode 100644 index 0000000000..3a8db22844 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/GreetingResource.java @@ -0,0 +1,16 @@ +package io.peerdb; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; + +@Path("/hello") +public class GreetingResource { + + @GET + @Produces(MediaType.TEXT_PLAIN) + public String hello() { + return "Hello from Quarkus REST"; + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java new file mode 100644 index 0000000000..a263ca80f1 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java @@ -0,0 +1,42 @@ +package io.peerdb.flow.jvm; + +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.StatusRuntimeException; +import io.quarkus.grpc.ExceptionHandler; +import io.quarkus.grpc.ExceptionHandlerProvider; +import io.quarkus.logging.Log; +import jakarta.enterprise.context.ApplicationScoped; + +@ApplicationScoped +public class DefaultExceptionHandlerProvider implements ExceptionHandlerProvider{ + public static boolean invoked; + + @Override + public ExceptionHandler createHandler(ServerCall.Listener listener, + ServerCall serverCall, Metadata metadata) { + return new HelloExceptionHandler<>(listener, serverCall, metadata); + } + + @Override + public Throwable transform(Throwable t) { + invoked = true; + Log.errorf(t, "Transforming exception"); + return ExceptionHandlerProvider.toStatusException(t, true); + } + + private static class HelloExceptionHandler extends ExceptionHandler { + public HelloExceptionHandler(ServerCall.Listener listener, ServerCall call, Metadata metadata) { + super(listener, call, metadata); + } + + @Override + protected void handleException(Throwable t, ServerCall call, Metadata metadata) { + invoked = true; + StatusRuntimeException sre = (StatusRuntimeException) ExceptionHandlerProvider.toStatusException(t, true); + Metadata trailers = sre.getTrailers() != null ? sre.getTrailers() : metadata; + call.close(sre.getStatus(), trailers); + } + } +} + diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/ErrorInterceptor.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/ErrorInterceptor.java new file mode 100644 index 0000000000..17a4068975 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/ErrorInterceptor.java @@ -0,0 +1,37 @@ +package io.peerdb.flow.jvm; + + +import io.grpc.*; +import io.quarkus.grpc.GlobalInterceptor; +import io.quarkus.logging.Log; +import jakarta.enterprise.context.ApplicationScoped; + +@ApplicationScoped +@GlobalInterceptor +public class ErrorInterceptor implements ServerInterceptor { + @Override + public ServerCall.Listener interceptCall(ServerCall serverCall, Metadata metadata, ServerCallHandler serverCallHandler) { + Log.infof("Intercepting call", metadata); + System.out.println("Intercepting call sout"); + + ServerCall listener = new ForwardingServerCall.SimpleForwardingServerCall(serverCall) { + @Override + public void sendMessage(RespT message) { + Log.debugf("Sending message for method: %s: {%s}", serverCall.getMethodDescriptor().getFullMethodName(), message); + super.sendMessage(message); + } + }; + + return new ForwardingServerCallListener.SimpleForwardingServerCallListener(serverCallHandler.startCall(listener, metadata)) { + + @Override + public void onMessage(ReqT message) { + Log.debugf("Received message from client for method: %s: {%s}", serverCall.getMethodDescriptor().getFullMethodName(), message); + super.onMessage(message); + } + + }; + +// return serverCallHandler.startCall(serverCall, metadata); + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/GrpcError.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/GrpcError.java new file mode 100644 index 0000000000..812a5af68a --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/GrpcError.java @@ -0,0 +1,11 @@ +package io.peerdb.flow.jvm; + +public class GrpcError extends RuntimeException { + public GrpcError(String message) { + super(message); + } + + public GrpcError(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java new file mode 100644 index 0000000000..037f11ccab --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java @@ -0,0 +1,61 @@ +package io.peerdb.flow.jvm.iceberg.avro; + +import io.quarkus.logging.Log; +import org.apache.avro.Schema; +import org.apache.avro.file.SeekableByteArrayInput; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DecoderFactory; +import org.apache.iceberg.data.Record; + +import java.io.IOException; + +public class AvroIcebergConverter { + private final org.apache.iceberg.Schema icebergSchema; + private final Schema avroSchema; + + public AvroIcebergConverter(String avroSchemaString, org.apache.iceberg.Schema icebergSchema) { + + this.icebergSchema = icebergSchema; + var avroSchemaParser = new org.apache.avro.Schema.Parser(); + this.avroSchema = avroSchemaParser.parse(avroSchemaString); + } + + public AvroIcebergConverter(Schema avroSchema, org.apache.iceberg.Schema icebergSchema) { + this.icebergSchema = icebergSchema; + this.avroSchema = avroSchema; + } + + public GenericRecord toAvroRecord(byte[] bytes) throws IOException { + var reader = new GenericDatumReader(avroSchema); + try (var byteStream = new SeekableByteArrayInput(bytes)) { + // The below code is for avro binary data files +// var dataFileReader = DataFileReader.openReader(byteStream, reader); +// if (!dataFileReader.hasNext()) { +// Log.errorf("No records found!"); +// return; +// } +// var record = dataFileReader.next(); + + var binaryDecoder = DecoderFactory.get().binaryDecoder(byteStream, null); + return reader.read(null, binaryDecoder); + } + } + + + public org.apache.iceberg.data.GenericRecord toIcebergRecord(GenericRecord avroRecord) { + var genericRecord = org.apache.iceberg.data.GenericRecord.create(icebergSchema); + for (var field : icebergSchema.columns()) { + var fieldName = field.name(); + var fieldValue = avroRecord.get(fieldName); + Log.tracef("Will set Field: %s, Value: %s, Current Record: %s", fieldName, fieldValue, genericRecord); + genericRecord.setField(fieldName, fieldValue); + } + return genericRecord; + } + + public org.apache.iceberg.data.GenericRecord toIcebergRecord(byte[] bytes) throws IOException { + return toIcebergRecord(toAvroRecord(bytes)); + } + +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java new file mode 100644 index 0000000000..6346a40127 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java @@ -0,0 +1,49 @@ +package io.peerdb.flow.jvm.iceberg.catalog; + + +import io.peerdb.flow.jvm.iceberg.catalog.io.mapper.S3IOConfigMapper; +import io.peerdb.flow.jvm.iceberg.catalog.mapper.HiveConfigMapper; +import io.peerdb.flow.jvm.iceberg.catalog.mapper.JdbcCatalogMapper; +import io.peerdb.flow.peers.IcebergCatalog; +import io.quarkus.logging.Log; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Catalog; + +import java.util.Collections; + + +@Singleton +public class CatalogLoader { + @Inject + HiveConfigMapper hiveConfigMapper; + + @Inject + JdbcCatalogMapper jdbcCatalogMapper; + + @Inject + S3IOConfigMapper s3IOConfigMapper; + + + + public Catalog loadCatalog(IcebergCatalog icebergCatalogConfig) { + var icebergIOConfig = icebergCatalogConfig.getIoConfig(); + var fileIoConfig = switch (icebergIOConfig.getConfigCase()) { + case S3 -> s3IOConfigMapper.map(icebergIOConfig.getS3()); + default -> { + Log.errorf("Unexpected value for file io config: %s", icebergIOConfig.getConfigCase()); + yield Collections.emptyMap(); + } + }; + + var catalogconfig = switch (icebergCatalogConfig.getConfigCase()) { + case HIVE -> hiveConfigMapper.map(icebergCatalogConfig.getCommonConfig(), icebergCatalogConfig.getHive(), fileIoConfig); + case JDBC -> jdbcCatalogMapper.map(icebergCatalogConfig.getCommonConfig(), icebergCatalogConfig.getJdbc(), fileIoConfig); + default -> throw new IllegalStateException("Unexpected value for catalog config: " + icebergCatalogConfig.getConfigCase()); + }; + // TODO look at hadoop + return CatalogUtil.buildIcebergCatalog(catalogconfig.get(CatalogProperties.CATALOG_IMPL), catalogconfig, null); + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FileIOConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FileIOConfigMapper.java new file mode 100644 index 0000000000..de64c10d27 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FileIOConfigMapper.java @@ -0,0 +1,28 @@ +package io.peerdb.flow.jvm.iceberg.catalog.io.mapper; + + +import io.quarkus.logging.Log; +import org.apache.iceberg.CatalogProperties; + +import java.util.HashMap; +import java.util.Map; + +public abstract class FileIOConfigMapper { + + protected abstract Map mapSpecific(T config); + + protected Map mapCommon() { + return Map.of( + CatalogProperties.FILE_IO_IMPL, this.implementationClass() + ); + } + + public final Map map(T config) { + var map = new HashMap<>(mapCommon()); + map.putAll(this.mapSpecific(config)); + Log.infof("Mapped IO config: %s", map); + return map; + } + + public abstract String implementationClass(); +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java new file mode 100644 index 0000000000..986e267902 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java @@ -0,0 +1,30 @@ +package io.peerdb.flow.jvm.iceberg.catalog.io.mapper; + +import com.google.common.collect.ImmutableMap; +import io.peerdb.flow.peers.IcebergS3IoConfig; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.iceberg.aws.s3.S3FileIOProperties; + +import java.util.Map; + +@ApplicationScoped +public class S3IOConfigMapper extends FileIOConfigMapper { + @Override + protected Map mapSpecific(IcebergS3IoConfig config) { + var builder = ImmutableMap.builder() + .put(S3FileIOProperties.ACCESS_KEY_ID, config.getAccessKeyId()) + .put(S3FileIOProperties.SECRET_ACCESS_KEY, config.getSecretAccessKey()); + if (config.hasEndpoint()) { + builder.put(S3FileIOProperties.ENDPOINT, config.getEndpoint()); + } + if (config.hasPathStyleAccess()) { + builder.put(S3FileIOProperties.PATH_STYLE_ACCESS, config.getPathStyleAccess()); + } + return builder.build(); + } + + @Override + public String implementationClass() { + return "org.apache.iceberg.aws.s3.S3FileIO"; + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java new file mode 100644 index 0000000000..709e6d9764 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java @@ -0,0 +1,40 @@ +package io.peerdb.flow.jvm.iceberg.catalog.mapper; + + +import com.google.common.collect.ImmutableMap; +import io.peerdb.flow.peers.CommonIcebergCatalog; +import io.quarkus.logging.Log; +import org.apache.iceberg.CatalogProperties; + +import java.util.HashMap; +import java.util.Map; + +public abstract class CatalogConfigMapper { + protected Map mapCommon(CommonIcebergCatalog config) { + var builder = ImmutableMap.builder() + .put(CatalogProperties.URI, config.getUri()) + .put(CatalogProperties.WAREHOUSE_LOCATION, config.getWarehouseLocation()) + .put(CatalogProperties.CATALOG_IMPL, this.implementationClass()); + if (config.hasClientPoolSize()) { + builder.put(CatalogProperties.CLIENT_POOL_SIZE, String.valueOf(config.getClientPoolSize())); + } + if (config.hasCacheEnabled()) { + builder.put(CatalogProperties.CACHE_ENABLED, String.valueOf(config.getCacheEnabled())); + } + builder.putAll(config.getAdditionalPropertiesMap()); + return builder.build(); + } + + public Map map(CommonIcebergCatalog commonConfig, T config, Map fileIoConfig) { + var map = new HashMap<>(mapCommon(commonConfig)); + map.putAll(this.mapSpecific(config)); + map.putAll(fileIoConfig); + Log.infof("Mapped catalog config: %s", map); + return map; + } + + + protected abstract Map mapSpecific(T config); + + public abstract String implementationClass(); +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java new file mode 100644 index 0000000000..c567425387 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java @@ -0,0 +1,21 @@ +package io.peerdb.flow.jvm.iceberg.catalog.mapper; + +import io.peerdb.flow.peers.HiveIcebergCatalog; +import jakarta.inject.Singleton; +import org.apache.iceberg.CatalogUtil; + +import java.util.Collections; +import java.util.Map; + +@Singleton +public class HiveConfigMapper extends CatalogConfigMapper { + @Override + protected Map mapSpecific(HiveIcebergCatalog config) { + return Collections.emptyMap(); + } + + @Override + public String implementationClass() { + return CatalogUtil.ICEBERG_CATALOG_HIVE; + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/JdbcCatalogMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/JdbcCatalogMapper.java new file mode 100644 index 0000000000..a1adc4c8f6 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/JdbcCatalogMapper.java @@ -0,0 +1,30 @@ +package io.peerdb.flow.jvm.iceberg.catalog.mapper; + +import io.peerdb.flow.peers.JdbcIcebergCatalog; +import jakarta.inject.Singleton; +import org.apache.iceberg.jdbc.JdbcCatalog; + +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; + + +@Singleton +public class JdbcCatalogMapper extends CatalogConfigMapper { + + @Override + protected Map mapSpecific(JdbcIcebergCatalog config) { + return Map.of( + "user", config.getUser(), + "password", config.getPassword(), + "useSSL", config.hasUseSsl() ? String.valueOf(config.getUseSsl()) : "true", + "verifyServerCertificate", config.hasVerifyServerCertificate() ? String.valueOf(config.getVerifyServerCertificate()) : "false" + + ).entrySet().stream().collect(Collectors.toMap(e -> JdbcCatalog.PROPERTY_PREFIX + e.getKey(), Map.Entry::getValue)); + } + + @Override + public String implementationClass() { + return JdbcCatalog.class.getName(); + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java new file mode 100644 index 0000000000..604390995a --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java @@ -0,0 +1,89 @@ +package io.peerdb.flow.jvm.iceberg.resource; + +import com.google.common.collect.Streams; +import io.peerdb.flow.jvm.grpc.*; +import io.peerdb.flow.jvm.iceberg.catalog.CatalogLoader; +import io.peerdb.flow.jvm.iceberg.service.IcebergService; +import io.quarkus.grpc.GrpcService; +import io.quarkus.logging.Log; +import io.smallrye.common.annotation.RunOnVirtualThread; +import io.smallrye.mutiny.Uni; +import jakarta.inject.Inject; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.IcebergGenerics; + +import java.io.IOException; +import java.util.Optional; + +@GrpcService +public class IcebergResource implements IcebergProxyService { + @Inject + CatalogLoader catalogLoader; + + @Inject + IcebergService icebergService; + + @RunOnVirtualThread + @Override + public Uni createTable(CreateTableRequest request) { + var tableInfo = request.getTableInfo(); + var icebergCatalog = tableInfo.getIcebergCatalog(); + + var catalog = catalogLoader.loadCatalog(icebergCatalog); + var icebergSchema = icebergService.getIcebergSchema(request.getSchema()); + var table = catalog.createTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName()), icebergSchema); + return Uni.createFrom().item(CreateTableResponse.newBuilder().setTableName(table.name()).build()); + } + + + @RunOnVirtualThread + @Override + public Uni dropTable(DropTableRequest request) { + var tableInfo = request.getTableInfo(); + var icebergCatalog = tableInfo.getIcebergCatalog(); + var catalog = catalogLoader.loadCatalog(icebergCatalog); + var droppedTable = catalog.dropTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName()), request.getPurge()); + return Uni.createFrom().item(DropTableResponse.newBuilder().setSuccess(droppedTable).build()); + + } + + @Override + public Uni countRecords(CountRecordRequest request) { + var tableInfo = request.getTableInfo(); + var icebergCatalog = tableInfo.getIcebergCatalog(); + var catalog = catalogLoader.loadCatalog(icebergCatalog); + var table = catalog.loadTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName())); + + Log.debugf("For table %s, schema is %s", tableInfo.getTableName(), table.schema()); + var count = 0L; + try (var tableScan = IcebergGenerics.read(table).build()) { + count = Streams.stream(tableScan.iterator()).reduce(0L, (current, record) -> current + 1L, Long::sum); + } catch (IOException e) { + Log.errorf(e, "Error reading table %s", tableInfo.getTableName()); + throw new RuntimeException(e); + } + return Uni.createFrom().item(CountRecordResponse.newBuilder().setCount(count).build()); + } + + @RunOnVirtualThread + @Override + public Uni insertChanges(InsertChangesRequest request) { + return Uni.createFrom() + .item(InsertChangesResponse.newBuilder() + .setSuccess( + icebergService.insertChanges( + request.getTableInfo(), + request.getSchema(), + request.getChangesList(), + Optional.ofNullable(request.hasBranchOptions() ? request.getBranchOptions() : null) + )) + .build()); + + } + + @Override + public Uni appendRecords(AppendRecordsRequest request) { + return Uni.createFrom().item(AppendRecordsResponse.newBuilder().setSuccess(icebergService.appendRecords(request.getTableInfo(), request.getSchema(), request.getRecordsList())).build()); + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java new file mode 100644 index 0000000000..94851007a4 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -0,0 +1,199 @@ +package io.peerdb.flow.jvm.iceberg.service; + +import io.peerdb.flow.jvm.grpc.BranchOptions; +import io.peerdb.flow.jvm.grpc.InsertRecord; +import io.peerdb.flow.jvm.grpc.RecordChange; +import io.peerdb.flow.jvm.grpc.TableInfo; +import io.peerdb.flow.jvm.iceberg.avro.AvroIcebergConverter; +import io.peerdb.flow.jvm.iceberg.catalog.CatalogLoader; +import io.quarkus.logging.Log; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.avro.AvroSchemaUtil; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.GenericAppenderFactory; +import org.apache.iceberg.io.OutputFileFactory; +import org.apache.iceberg.io.UnpartitionedWriter; +import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.util.PropertyUtil; + +import java.io.IOException; +import java.util.*; + +import static org.apache.iceberg.TableProperties.WRITE_TARGET_FILE_SIZE_BYTES; +import static org.apache.iceberg.TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT; + +@ApplicationScoped +public class IcebergService { + + @Inject + CatalogLoader catalogLoader; + + + public boolean insertChanges(TableInfo tableInfo, String avroSchema, List recordChanges, Optional branchOptions) { + var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); + var table = icebergCatalog.loadTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName())); + if (branchOptions.isPresent()) { + var branchName = branchOptions.get().getBranch(); + if (table.refs().containsKey(branchName)) { + switch (branchOptions.get().getBranchCreateConflictPolicy()) { + case ERROR -> + throw new IllegalArgumentException(String.format("Branch %s already exists", branchName)); + case IGNORE -> { + return false; + } + case DROP -> table.newTransaction().manageSnapshots().removeBranch(branchName).commit(); + default -> throw new IllegalArgumentException(String.format("Unrecognized branch create conflict policy %s", branchOptions.get().getBranchCreateConflictPolicy())); + } + } + } + var appenderFactory = new GenericAppenderFactory(table.schema(), table.spec()); + var format = FileFormat.fromString(table.properties().getOrDefault(TableProperties.DEFAULT_FILE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)); + var outputFileFactory = OutputFileFactory.builderFor(table, 1, System.currentTimeMillis()) + .defaultSpec(table.spec()) + .operationId(UUID.randomUUID().toString()) + .format(format) + .build(); + var targetFileSize = PropertyUtil.propertyAsLong(table.properties(), WRITE_TARGET_FILE_SIZE_BYTES, WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT); + + var converter = new AvroIcebergConverter(avroSchema, table.schema()); + var writer = new UnpartitionedWriter<>(table.spec(), format, appenderFactory, outputFileFactory, table.io(), targetFileSize); + recordChanges.forEach(recordChange -> { + switch (recordChange.getChangeCase()) { + case INSERT: + Log.tracef("Inserting record: %s", recordChange.getInsert()); + var insertRecord = recordChange.getInsert(); + try { + var genericRecord = converter.toIcebergRecord(insertRecord.getRecord().toByteArray()); + } catch (IOException e) { + Log.errorf(e, "Error while converting record"); + throw new RuntimeException(e); + } + + break; + case DELETE: + Log.tracef("Deleting record: %s", recordChange.getDelete()); + var deleteRecord = recordChange.getDelete(); + break; + case UPDATE: + Log.tracef("Updating record: %s", recordChange.getUpdate()); + var updateRecord = recordChange.getUpdate(); + break; + } +// try { +// var encoded = Base64.getEncoder().encodeToString(recordChange.getRecord().toByteArray()); +// Log.tracef("Received record: %s", encoded); +// converter.toIcebergRecord(recordChange.getRecord().toByteArray()); +// var genericRecord = GenericRecord.create(table.schema()); +// Log.tracef("Writing record: %s", genericRecord); +// writer.write(genericRecord); +// } catch (IOException e) { +// Log.errorf(e, "Error while converting/writing record"); +// throw new RuntimeException(e); +// } + }); + + + + WriteResult writeResult; + try { + writeResult = writer.complete(); + } catch (IOException e) { + Log.errorf(e, "Error while completing writing records"); + throw new RuntimeException(e); + } + + var transaction = table.newTransaction(); + branchOptions.ifPresent(options -> transaction.manageSnapshots().createBranch(options.getBranch()) +// .setMaxRefAgeMs() +// .setMinSnapshotsToKeep() +// .setMaxSnapshotAgeMs() + .commit()); + + + var appendFiles = transaction.newAppend(); + + if (branchOptions.isPresent()) { + appendFiles = appendFiles.toBranch(branchOptions.get().getBranch()); + } + + Arrays.stream(writeResult.dataFiles()).forEach(appendFiles::appendFile); + appendFiles.commit(); + transaction.commitTransaction(); + return false; + } + + public Schema getIcebergSchema(String schemaString) { + var avroSchemaParser = new org.apache.avro.Schema.Parser(); + var avroSchema = avroSchemaParser.parse(schemaString); + return AvroSchemaUtil.toIceberg(avroSchema); + } + + public boolean appendRecords(TableInfo tableInfo, String avroSchema, List insertRecords) { + var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); + var table = icebergCatalog.loadTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName())); + + // TODO add support for identifier fields + var appenderFactory = new GenericAppenderFactory(table.schema(), table.spec()); + var format = FileFormat.fromString(table.properties().getOrDefault(TableProperties.DEFAULT_FILE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)); + // TODO add partition ID + var outputFileFactory = OutputFileFactory.builderFor(table, 1, System.currentTimeMillis()) + .defaultSpec(table.spec()) + .operationId(UUID.randomUUID().toString()) + .format(format) + .build(); + var targetFileSize = PropertyUtil.propertyAsLong(table.properties(), WRITE_TARGET_FILE_SIZE_BYTES, WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT); + + var writer = new UnpartitionedWriter<>(table.spec(), format, appenderFactory, outputFileFactory, table.io(), targetFileSize); + var converter = new AvroIcebergConverter(avroSchema, table.schema()); + // TODO can make this parallel for speed up + insertRecords.forEach(insertRecord -> { + try { + var encoded = Base64.getEncoder().encodeToString(insertRecord.getRecord().toByteArray()); + Log.tracef("Received record: %s", encoded); + var genericRecord = converter.toIcebergRecord(insertRecord.getRecord().toByteArray()); + Log.tracef("Writing record: %s", genericRecord); + writer.write(genericRecord); + } catch (IOException e) { + Log.errorf(e, "Error while converting/writing record"); + throw new RuntimeException(e); + } + }); + WriteResult writeResult; + try { + writeResult = writer.complete(); + } catch (IOException e) { + Log.errorf(e, "Error while completing writing records"); + throw new RuntimeException(e); + } + + var branchName = "_flow_something_something_temp"; + + if (table.refs().containsKey(branchName)) { + // Maybe skip everything and just return (depending on input) to skip to normalize + // Or maybe we should drop and redo everything (as there might be some dirty tables) + } + + var transaction = table.newTransaction(); +// transaction.manageSnapshots().createBranch(branchName) +//// .setMaxRefAgeMs() +//// .setMinSnapshotsToKeep() +//// .setMaxSnapshotAgeMs() +// .commit();; + + + var appendFiles = transaction.newAppend() +// .toBranch(branchName) + ; + + Arrays.stream(writeResult.dataFiles()).forEach(appendFiles::appendFile); + appendFiles.commit(); + transaction.commitTransaction(); + + return true; + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/TaskWriter.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/TaskWriter.java new file mode 100644 index 0000000000..596b622dc3 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/TaskWriter.java @@ -0,0 +1,4 @@ +//package io.peerdb.flow.jvm.iceberg.writer; +// +//public class UnpartitionedTaskWriter extends { +//} diff --git a/flow-jvm/src/main/resources/application.properties b/flow-jvm/src/main/resources/application.properties new file mode 100644 index 0000000000..f0b85408ff --- /dev/null +++ b/flow-jvm/src/main/resources/application.properties @@ -0,0 +1 @@ +# USE applciation.yaml instead diff --git a/flow-jvm/src/main/resources/application.yaml b/flow-jvm/src/main/resources/application.yaml new file mode 100644 index 0000000000..5b95921eef --- /dev/null +++ b/flow-jvm/src/main/resources/application.yaml @@ -0,0 +1,24 @@ +quarkus: + generate-code: + grpc: + scan-for-imports: all + grpc: + server: + port: ${FLOW_JVM_GRPC_PORT:9801} + + # 512 MB + max-inbound-message-size: 536870912 + log: + level: INFO + category: + "io.peerdb": + level: ${PEERDB_LOG_LEVEL:INFO} + min-level: ${PEERDB_LOG_LEVEL:TRACE} + console: + level: ${PEERDB_LOG_LEVEL:TRACE} + http: + port: ${FLOW_JVM_HTTP_PORT:9802} + + test: + continuous-testing: + enabled: disabled diff --git a/flow-jvm/src/native-test/java/io/peerdb/GreetingResourceIT.java b/flow-jvm/src/native-test/java/io/peerdb/GreetingResourceIT.java new file mode 100644 index 0000000000..e5d0418077 --- /dev/null +++ b/flow-jvm/src/native-test/java/io/peerdb/GreetingResourceIT.java @@ -0,0 +1,8 @@ +package io.peerdb; + +import io.quarkus.test.junit.QuarkusIntegrationTest; + +@QuarkusIntegrationTest +class GreetingResourceIT extends GreetingResourceTest { + // Execute the same tests but in packaged mode. +} diff --git a/flow-jvm/src/test/java/io/peerdb/GreetingResourceTest.java b/flow-jvm/src/test/java/io/peerdb/GreetingResourceTest.java new file mode 100644 index 0000000000..e73353ad0a --- /dev/null +++ b/flow-jvm/src/test/java/io/peerdb/GreetingResourceTest.java @@ -0,0 +1,20 @@ +package io.peerdb; + +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.Test; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.CoreMatchers.is; + +@QuarkusTest +class GreetingResourceTest { + @Test + void testHelloEndpoint() { + given() + .when().get("/hello") + .then() + .statusCode(200) + .body(is("Hello from Quarkus REST")); + } + +} \ No newline at end of file diff --git a/flow-jvm/src/test/java/io/peerdb/IcebergServiceTest.java b/flow-jvm/src/test/java/io/peerdb/IcebergServiceTest.java new file mode 100644 index 0000000000..29e8193f60 --- /dev/null +++ b/flow-jvm/src/test/java/io/peerdb/IcebergServiceTest.java @@ -0,0 +1,22 @@ +//package io.peerdb; +// +//import io.peerdb.flow.jvm.iceberg.resource.IcebergResource; +//import io.peerdb.flow.jvm.iceberg.service.IcebergService; +//import io.quarkus.grpc.GrpcClient; +//import io.quarkus.test.junit.QuarkusTest; +//import org.junit.jupiter.api.Test; +// +//@QuarkusTest +//public class IcebergServiceTest { +// @GrpcClient +// IcebergService client; +// +// @Test +// public void printRecordCount() { +// var response = client.createTable(null).onFailure().invoke(e -> { +// System.out.println(e); +// }).onFailure().recoverWithNull().await().indefinitely(); +// System.out.println(response); +// } +// +//} diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go index 844c60c088..b866f6be67 100644 --- a/flow/activities/flowable.go +++ b/flow/activities/flowable.go @@ -919,6 +919,13 @@ func (a *FlowableActivity) LoadPeer(ctx context.Context, peerName string) (*prot return nil, fmt.Errorf("failed to unmarshal Elasticsearch config: %w", err) } peer.Config = &protos.Peer_ElasticsearchConfig{ElasticsearchConfig: &config} + case protos.DBType_ICEBERG: + var config protos.IcebergConfig + if err := proto.Unmarshal(peerOptions, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal Iceberg config: %w", err) + + } + peer.Config = &protos.Peer_IcebergConfig{IcebergConfig: &config} default: return nil, fmt.Errorf("unsupported peer type: %s", peer.Type) } diff --git a/flow/connectors/core.go b/flow/connectors/core.go index 0f25c4509c..5200d742d4 100644 --- a/flow/connectors/core.go +++ b/flow/connectors/core.go @@ -3,6 +3,7 @@ package connectors import ( "context" "errors" + "github.com/PeerDB-io/peer-flow/connectors/iceberg" "log/slog" "github.com/jackc/pgx/v5/pgxpool" @@ -271,6 +272,8 @@ func GetConnector(ctx context.Context, config *protos.Peer) (Connector, error) { return connpubsub.NewPubSubConnector(ctx, inner.PubsubConfig) case *protos.Peer_ElasticsearchConfig: return connelasticsearch.NewElasticsearchConnector(ctx, inner.ElasticsearchConfig) + case *protos.Peer_IcebergConfig: + return iceberg.NewIcebergConnector(ctx, inner.IcebergConfig) default: return nil, errors.ErrUnsupported } @@ -336,6 +339,7 @@ var ( _ CDCSyncConnector = &conns3.S3Connector{} _ CDCSyncConnector = &connclickhouse.ClickhouseConnector{} _ CDCSyncConnector = &connelasticsearch.ElasticsearchConnector{} + _ CDCSyncConnector = &iceberg.IcebergConnector{} _ CDCSyncPgConnector = &connpostgres.PostgresConnector{} @@ -364,6 +368,7 @@ var ( _ QRepSyncConnector = &conns3.S3Connector{} _ QRepSyncConnector = &connclickhouse.ClickhouseConnector{} _ QRepSyncConnector = &connelasticsearch.ElasticsearchConnector{} + _ QRepSyncConnector = &iceberg.IcebergConnector{} _ QRepSyncPgConnector = &connpostgres.PostgresConnector{} @@ -377,6 +382,7 @@ var ( _ ValidationConnector = &connclickhouse.ClickhouseConnector{} _ ValidationConnector = &connbigquery.BigQueryConnector{} _ ValidationConnector = &conns3.S3Connector{} + _ ValidationConnector = &iceberg.IcebergConnector{} _ Connector = &connmysql.MySqlConnector{} ) diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go new file mode 100644 index 0000000000..9796e20ad2 --- /dev/null +++ b/flow/connectors/iceberg/iceberg.go @@ -0,0 +1,302 @@ +package iceberg + +import ( + "context" + "fmt" + "github.com/PeerDB-io/peer-flow/alerting" + "github.com/PeerDB-io/peer-flow/datatypes" + "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/otel_metrics/peerdb_guages" + "github.com/jackc/pgx/v5/pgxpool" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "log/slog" + "strconv" + "time" + + "go.temporal.io/sdk/log" + + metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" + "github.com/PeerDB-io/peer-flow/connectors/utils" + "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/model" +) + +type IcebergConnector struct { + *metadataStore.PostgresMetadata + logger log.Logger + config *protos.IcebergConfig + grpcConnection *grpc.ClientConn + proxyClient protos.IcebergProxyServiceClient +} + +func (c *IcebergConnector) GetTableSchema(ctx context.Context, req *protos.GetTableSchemaBatchInput) (*protos.GetTableSchemaBatchOutput, error) { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) EnsurePullability(ctx context.Context, req *protos.EnsurePullabilityBatchInput) (*protos.EnsurePullabilityBatchOutput, error) { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) ExportTxSnapshot(ctx context.Context) (*protos.ExportTxSnapshotOutput, any, error) { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) FinishExport(a any) error { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) SetupReplConn(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) ReplPing(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) UpdateReplStateLastOffset(lastOffset int64) { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) PullFlowCleanup(ctx context.Context, jobName string) error { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) HandleSlotInfo(ctx context.Context, alerter *alerting.Alerter, catalogPool *pgxpool.Pool, slotName string, peerName string, slotMetricGuages peerdb_guages.SlotMetricGuages) error { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) GetSlotInfo(ctx context.Context, slotName string) ([]*protos.SlotInfo, error) { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) AddTablesToPublication(ctx context.Context, req *protos.AddTablesToPublicationInput) error { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) PullRecords(ctx context.Context, catalogPool *pgxpool.Pool, req *model.PullRecordsRequest[model.RecordItems]) error { + //TODO implement me + panic("implement me") +} + +func (c *IcebergConnector) StartSetupNormalizedTables(ctx context.Context) (any, error) { + return nil, nil +} + +func (c *IcebergConnector) CleanupSetupNormalizedTables(ctx context.Context, tx any) { + return + +} + +func (c *IcebergConnector) FinishSetupNormalizedTables(ctx context.Context, tx any) error { + return nil +} + +func (c *IcebergConnector) SyncFlowCleanup(ctx context.Context, jobName string) error { + err := c.PostgresMetadata.SyncFlowCleanup(ctx, jobName) + if err != nil { + return fmt.Errorf("unable to clear metadata for sync flow cleanup : %w", err) + } + // TODO implement this + c.logger.Debug("SyncFlowCleanup for Iceberg is a no-op") + return nil +} + +func (c *IcebergConnector) SetupNormalizedTable(ctx context.Context, tx any, tableIdentifier string, tableSchema *protos.TableSchema, softDeleteColName string, syncedAtColName string) (bool, error) { + // TODO add soft delete column in the schema + qFields := make([]qvalue.QField, len(tableSchema.Columns)) + for i, fieldDescription := range tableSchema.Columns { + colName := fieldDescription.Name + qValueKind := qvalue.QValueKind(fieldDescription.Type) + var precision, scale int16 + if qValueKind == qvalue.QValueKindNumeric { + precision, scale = datatypes.ParseNumericTypmod(fieldDescription.TypeModifier) + } + qField := qvalue.QField{ + Name: colName, + Type: qValueKind, + Precision: precision, + Scale: scale, + // TODO check this + Nullable: true, + } + qFields[i] = qField + } + + qFields = append(qFields, qvalue.QField{ + Name: softDeleteColName, + Type: qvalue.QValueKindBoolean, + Nullable: true, + }) + + qFields = append(qFields, qvalue.QField{ + Name: syncedAtColName, + Type: qvalue.QValueKindTimestamp, + Nullable: true, + }) + + avroSchema, err := getAvroSchema(tableIdentifier, qvalue.NewQRecordSchema(qFields)) + if err != nil { + return false, err + } + + // TODO save to a buffer and call when Finish is called + // TODO maybe can later migrate to a streaming rpc with transaction support + tableResponse, err := c.proxyClient.CreateTable(ctx, &protos.CreateTableRequest{ + TableInfo: &protos.TableInfo{ + Namespace: nil, + TableName: tableIdentifier, + IcebergCatalog: c.config.CatalogConfig, + PrimaryKey: tableSchema.GetPrimaryKeyColumns(), + }, + Schema: avroSchema.Schema, + }) + if err != nil { + return false, err + } + c.logger.Debug("Created iceberg table", slog.String("table", tableResponse.TableName)) + // TODO need to re-enable this and see why it is failing + //if tableResponse.TableName != tableIdentifier { + // return false, fmt.Errorf("created table name mismatch: %s != %s", tableResponse.TableName, tableIdentifier) + //} + return true, nil +} + +func NewIcebergConnector( + ctx context.Context, + config *protos.IcebergConfig, +) (*IcebergConnector, error) { + logger := logger.LoggerFromCtx(ctx) + conn, err := grpc.NewClient("localhost:9801", grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("failed to connect to Iceberg proxy: %w", err) + } + client := protos.NewIcebergProxyServiceClient(conn) + + pgMetadata, err := metadataStore.NewPostgresMetadata(ctx) + if err != nil { + logger.Error("failed to create postgres metadata store", "error", err) + return nil, err + } + + return &IcebergConnector{ + PostgresMetadata: pgMetadata, + logger: logger, + config: config, + grpcConnection: conn, + proxyClient: client, + }, nil +} + +func (c *IcebergConnector) CreateRawTable(_ context.Context, req *protos.CreateRawTableInput) (*protos.CreateRawTableOutput, error) { + // TODO + c.logger.Info("CreateRawTable for S3 is a no-op") + return nil, nil +} + +func (c *IcebergConnector) Close() error { + return c.grpcConnection.Close() +} + +func (c *IcebergConnector) ValidateCheck(ctx context.Context) error { + // Create a table with a random name based on current time + tableName := fmt.Sprintf("peerdb_test_flow_%d", time.Now().Unix()) + c.logger.Debug("Will try to create iceberg table", "table", tableName) + table, err := c.proxyClient.CreateTable(ctx, + &protos.CreateTableRequest{ + TableInfo: &protos.TableInfo{ + Namespace: nil, + TableName: tableName, + IcebergCatalog: c.config.CatalogConfig, + PrimaryKey: nil, + }, + Schema: "{\n \"type\": \"record\",\n \"name\": \"TestObject\",\n \"namespace\": \"ca.dataedu\",\n \"fields\": [\n {\n \"name\": \"hello\",\n \"type\": [\n \"null\",\n \"int\"\n ],\n \"default\": null\n },\n {\n \"name\": \"some\",\n \"type\": [\n \"null\",\n \"string\"\n ],\n \"default\": null\n }\n ]\n}", + }) + if err != nil { + return err + } + c.logger.Debug("Created iceberg table", slog.String("table", table.TableName)) + c.logger.Debug("Will try to drop iceberg table", "table", tableName) + dropTable, err := c.proxyClient.DropTable(ctx, + &protos.DropTableRequest{ + TableInfo: &protos.TableInfo{ + Namespace: nil, + TableName: tableName, + IcebergCatalog: c.config.CatalogConfig, + PrimaryKey: nil, + }, + Purge: true, + }, + ) + if err != nil { + return err + } + if !dropTable.Success { + return fmt.Errorf("failed to drop table %s", tableName) + } + c.logger.Debug("Dropped iceberg table", slog.String("table", tableName)) + return nil +} + +func (c *IcebergConnector) ConnectionActive(ctx context.Context) error { + // TODO + return nil +} + +func (c *IcebergConnector) SyncRecords(ctx context.Context, req *model.SyncRecordsRequest[model.RecordItems]) (*model.SyncResponse, error) { + tableNameRowsMapping := utils.InitialiseTableRowsMap(req.TableMappings) + streamReq := model.NewRecordsToStreamRequest(req.Records.GetRecords(), tableNameRowsMapping, req.SyncBatchID) + records := req.Records.GetRecords() + a := <-records + + items := a.GetItems() + qValue := items.GetColumnValue("asd") + req.TableNameSchemaMapping["asd"].Columns + + if err != nil { + return nil, fmt.Errorf("failed to convert records to raw table stream: %w", err) + } + qrepConfig := &protos.QRepConfig{ + FlowJobName: req.FlowJobName, + DestinationTableIdentifier: "raw_table_" + req.FlowJobName, + } + partition := &protos.QRepPartition{ + PartitionId: strconv.FormatInt(req.SyncBatchID, 10), + } + numRecords, err := c.SyncQRepRecords(ctx, qrepConfig, partition, recordStream) + if err != nil { + return nil, err + } + c.logger.Info(fmt.Sprintf("Synced %d records", numRecords)) + + lastCheckpoint := req.Records.GetLastCheckpoint() + err = c.FinishBatch(ctx, req.FlowJobName, req.SyncBatchID, lastCheckpoint) + if err != nil { + c.logger.Error("failed to increment id", "error", err) + return nil, err + } + + return &model.SyncResponse{ + LastSyncedCheckpointID: lastCheckpoint, + NumRecordsSynced: int64(numRecords), + TableNameRowsMapping: tableNameRowsMapping, + TableSchemaDeltas: req.Records.SchemaDeltas, + }, nil +} + +func (c *IcebergConnector) ReplayTableSchemaDeltas(_ context.Context, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error { + c.logger.Info("ReplayTableSchemaDeltas for S3 is a no-op") + return nil +} diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go new file mode 100644 index 0000000000..34f15b42f6 --- /dev/null +++ b/flow/connectors/iceberg/qrep.go @@ -0,0 +1,149 @@ +package iceberg + +import ( + "context" + "fmt" + "github.com/PeerDB-io/peer-flow/logger" + "github.com/linkedin/goavro/v2" + "log/slog" + + "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/model/qvalue" +) + +func (c *IcebergConnector) SyncQRepRecords( + ctx context.Context, + config *protos.QRepConfig, + partition *protos.QRepPartition, + stream *model.QRecordStream, +) (int, error) { + schema := stream.Schema() + + dstTableName := config.DestinationTableIdentifier + + avroSchema, err := getAvroSchema(dstTableName, schema) + if err != nil { + return 0, err + } + + avroConverter := model.NewQRecordAvroConverter( + avroSchema, + protos.DBType_ICEBERG, + schema.GetColumnNames(), + logger.LoggerFromCtx(ctx), + ) + codec, err := goavro.NewCodec(avroSchema.Schema) + if err != nil { + return 0, fmt.Errorf("failed to create Avro codec: %w", err) + } + binaryRecords := make([]*protos.InsertRecord, 0) + for record := range stream.Records { + converted, err := avroConverter.Convert(record) + if err != nil { + return 0, fmt.Errorf("failed to convert QRecord to Avro-compatible map: %w", err) + } + binaryData := make([]byte, 0) + native, err := codec.BinaryFromNative(binaryData, converted) + if err != nil { + return 0, fmt.Errorf("failed to convert Avro map to binary: %w", err) + } + binaryRecords = append(binaryRecords, &protos.InsertRecord{ + Record: native, + }) + + } + + appendRecordsResponse, err := c.proxyClient.AppendRecords(ctx, + &protos.AppendRecordsRequest{ + TableInfo: &protos.TableInfo{ + //Namespace: nil, + TableName: dstTableName, + IcebergCatalog: c.config.CatalogConfig, + //PrimaryKey: nil, + }, + Schema: avroSchema.Schema, + Records: binaryRecords, + }, + ) + + if err != nil { + return 0, err + } + + logger.LoggerFromCtx(ctx).Info("AppendRecordsResponse", slog.Any("response", appendRecordsResponse)) + + //numRecords, err := c.writeToAvroFile(ctx, stream, avroSchema, partition.PartitionId, config.FlowJobName) + //if err != nil { + // return 0, err + //} + + return len(binaryRecords), nil +} + +//func (c *IcebergConnector) writeToIceberg( +// ctx context.Context, +// stream *model.QRecordStream, +// avroSchema *model.QRecordAvroSchemaDefinition, +// destinationTableName string, +//) (int, error) { +// c.proxyClient.InsertChanges(ctx, &protos.InsertChangesRequest{ +// TableInfo: &protos.TableInfo{ +// Namespace: nil, +// TableName: destinationTableName, +// IcebergCatalog: c.config.CatalogConfig, +// PrimaryKey:, +// }, +// Schema: avroSchema.Schema, +// Changes: nil, +// }) +//} + +func getAvroSchema( + dstTableName string, + schema qvalue.QRecordSchema, +) (*model.QRecordAvroSchemaDefinition, error) { + avroSchema, err := model.GetAvroSchemaDefinition(dstTableName, schema, protos.DBType_ICEBERG) + if err != nil { + return nil, fmt.Errorf("failed to define Avro schema: %w", err) + } + + return avroSchema, nil +} + +//func (c *IcebergConnector) writeToAvroFile( +// ctx context.Context, +// stream *model.QRecordStream, +// avroSchema *model.QRecordAvroSchemaDefinition, +// partitionID string, +// jobName string, +//) (int, error) { +// s3o, err := utils.NewS3BucketAndPrefix(c.url) +// if err != nil { +// return 0, fmt.Errorf("failed to parse bucket path: %w", err) +// } +// +// s3AvroFileKey := fmt.Sprintf("%s/%s/%s.avro", s3o.Prefix, jobName, partitionID) +// +// writer := avro.NewPeerDBOCFWriter(stream, avroSchema, avro.CompressNone, protos.DBType_SNOWFLAKE) +// avroFile, err := writer.WriteRecordsToS3(ctx, s3o.Bucket, s3AvroFileKey, c.credentialsProvider) +// if err != nil { +// return 0, fmt.Errorf("failed to write records to S3: %w", err) +// } +// defer avroFile.Cleanup() +// +// return avroFile.NumRecords, nil +//} + +// S3 just sets up destination, not metadata tables +func (c *IcebergConnector) SetupQRepMetadataTables(_ context.Context, config *protos.QRepConfig) error { + c.logger.Info("QRep metadata setup not needed for S3.") + return nil +} + +// S3 doesn't check if partition is already synced, but file with same name is overwritten +func (c *IcebergConnector) IsQRepPartitionSynced(_ context.Context, + config *protos.IsQRepPartitionSyncedInput, +) (bool, error) { + return false, nil +} diff --git a/flow/connectors/utils/peers.go b/flow/connectors/utils/peers.go index 92947e80fa..bd68f54c3d 100644 --- a/flow/connectors/utils/peers.go +++ b/flow/connectors/utils/peers.go @@ -96,6 +96,13 @@ func CreatePeerNoValidate( } esConfig := esConfigObject.ElasticsearchConfig encodedConfig, encodingErr = proto.Marshal(esConfig) + case protos.DBType_ICEBERG: + icebergConfigObject, ok := config.(*protos.Peer_IcebergConfig) + if !ok { + return wrongConfigResponse, nil + } + icebergConfig := icebergConfigObject.IcebergConfig + encodedConfig, encodingErr = proto.Marshal(icebergConfig) default: return wrongConfigResponse, nil } @@ -106,9 +113,9 @@ func CreatePeerNoValidate( } _, err := pool.Exec(ctx, ` - INSERT INTO peers (name, type, options) + INSERT INTO peers (name, type, options) VALUES ($1, $2, $3) - ON CONFLICT (name) DO UPDATE + ON CONFLICT (name) DO UPDATE SET type = $2, options = $3`, peer.Name, peerType, encodedConfig, ) diff --git a/flow/logger/handler.go b/flow/logger/handler.go index 3351787228..8103fe7d4d 100644 --- a/flow/logger/handler.go +++ b/flow/logger/handler.go @@ -23,6 +23,9 @@ func NewHandler(handler slog.Handler) slog.Handler { } func (h Handler) Enabled(ctx context.Context, level slog.Level) bool { + if level == slog.LevelDebug { + return true + } return h.handler.Enabled(ctx, level) } diff --git a/protos/flow-jvm.proto b/protos/flow-jvm.proto new file mode 100644 index 0000000000..ed30c4a207 --- /dev/null +++ b/protos/flow-jvm.proto @@ -0,0 +1,106 @@ +syntax = "proto3"; + +package peerdb_flow_jvm; + +import "peers.proto"; + +option java_multiple_files = true; +option java_package = "io.peerdb.flow.jvm.grpc"; + +service IcebergProxyService { + rpc CreateTable(CreateTableRequest) returns (CreateTableResponse) {} + + rpc DropTable(DropTableRequest) returns (DropTableResponse) {} + + rpc CountRecords(CountRecordRequest) returns (CountRecordResponse) {} + + rpc InsertChanges(InsertChangesRequest) returns (InsertChangesResponse) {} + + rpc AppendRecords(AppendRecordsRequest) returns (AppendRecordsResponse) {} +} + + +message TableInfo { + repeated string namespace = 1; + string table_name = 2; + peerdb_peers.IcebergCatalog iceberg_catalog = 3; + repeated string primary_key = 4; +} + +message CreateTableRequest { + TableInfo table_info = 1; + string schema = 2; +} + +message CreateTableResponse { + string table_name = 1; +} + +message DropTableRequest { + TableInfo table_info = 1; + bool purge = 2; +} + +message DropTableResponse { + bool success = 1; +} + + +message CountRecordRequest { + TableInfo table_info = 1; +} + +message CountRecordResponse { + int64 count = 1; +} + +message BranchOptions { + string branch = 1; + BranchCreateConflictPolicy branch_create_conflict_policy = 2; +} + +enum BranchCreateConflictPolicy { + ERROR = 0; + DROP = 1; + IGNORE = 2; +} + +message InsertChangesRequest { + TableInfo table_info = 1; + string schema = 2; + repeated RecordChange changes = 3; + optional BranchOptions branch_options = 4; +} + +message InsertRecord { + bytes record = 1; +} + +message DeleteRecord { + bytes record = 1; +} + +message UpdateRecord { + bytes record = 1; +} + +message RecordChange { + oneof change { + InsertRecord insert = 1; + DeleteRecord delete = 2; + UpdateRecord update = 3; + } +} +message InsertChangesResponse { + bool success = 1; +} + +message AppendRecordsRequest { + TableInfo table_info = 1; + string schema = 2; + repeated InsertRecord records = 3; +} + +message AppendRecordsResponse { + bool success = 1; +} diff --git a/protos/peers.proto b/protos/peers.proto index 13cabf58cf..040cb29dbb 100644 --- a/protos/peers.proto +++ b/protos/peers.proto @@ -2,6 +2,10 @@ syntax = "proto3"; package peerdb_peers; +option java_multiple_files = true; +option java_package = "io.peerdb.flow.peers"; + + message SSHConfig { string host = 1; uint32 port = 2; @@ -162,6 +166,64 @@ message ElasticsearchConfig { optional string api_key = 5; } +message IcebergConfig { + IcebergCatalog catalog_config = 1; +} + +message IcebergS3IoConfig { + optional string access_key_id = 1; + optional string secret_access_key = 2; + optional string region = 3; + optional string endpoint = 4; + // Set to true to use for services like MinIO + optional string path_style_access = 5; +} + +message IcebergIOConfig { + oneof config { + IcebergS3IoConfig s3 = 1; + } +} + +message CommonIcebergCatalog { + string uri = 1; + string warehouse_location = 2; + optional int32 client_pool_size = 3; + optional bool cache_enabled = 4; + map additional_properties = 5; +} + +message HiveIcebergCatalog {} + +message HadoopIcebergCatalog {} + +message RestIcebergCatalog {} + +message GlueIcebergCatalog {} + +message JdbcIcebergCatalog { + optional string user = 1; + optional string password = 2; + optional bool use_ssl = 3; + optional bool verify_server_certificate = 4; +} + +message NessieIcebergCatalog {} + +message IcebergCatalog { + CommonIcebergCatalog common_config = 1; + peerdb_peers.IcebergIOConfig io_config = 2; + oneof config { + HiveIcebergCatalog hive = 3; + HadoopIcebergCatalog hadoop = 4; + RestIcebergCatalog rest = 5; + GlueIcebergCatalog glue = 6; + JdbcIcebergCatalog jdbc = 7; + NessieIcebergCatalog nessie = 8; + } +} + + enum DBType { BIGQUERY = 0; SNOWFLAKE = 1; @@ -175,6 +237,7 @@ enum DBType { PUBSUB = 10; EVENTHUBS = 11; ELASTICSEARCH = 12; + ICEBERG = 13; } message Peer { @@ -193,5 +256,6 @@ message Peer { PubSubConfig pubsub_config = 13; ElasticsearchConfig elasticsearch_config = 14; MySqlConfig mysql_config = 15; + IcebergConfig iceberg_config = 16; } } From 94fffcd828bb122eb0a898730b817d7024b88623 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 11 Jun 2024 09:16:48 +0530 Subject: [PATCH 02/31] feat: Enhance Iceberg support and refactor codebase - Introduce significant enhancements to Iceberg support, including the handling of Iceberg tables and improved JVM compatibility. - Refactor codebase to improve readability and maintainability, including the extraction of common functionality into separate methods and the removal of redundant code. - Implement new methods for handling Iceberg tables, including table creation, record appending, and table dropping. - Introduce a new `RecordWriterFactory` to manage record writing tasks. - Update `build.gradle` and `application.yaml` files to reflect changes in the project setup. - Remove unused test files and update the logging system for better debugging. - Update Go files in the `flow/connectors/iceberg` directory to improve Iceberg handling. - Add new protobuf messages in `protos/peers.proto` for Google Cloud Storage (GCS) configuration. - Update `protos/flow-jvm.proto` to include an idempotency key in the `AppendRecordsRequest` message. --- flow-jvm/build.gradle | 10 +- flow-jvm/gradle.properties | 4 +- .../main/java/io/peerdb/GreetingResource.java | 16 - .../jvm/DefaultExceptionHandlerProvider.java | 17 +- .../io/peerdb/flow/jvm/ErrorInterceptor.java | 37 --- .../java/io/peerdb/flow/jvm/GrpcError.java | 11 - .../flow/jvm/RequestLoggingInterceptor.java | 49 +++ .../iceberg/avro/AvroIcebergConverter.java | 54 ++-- .../jvm/iceberg/catalog/CatalogLoader.java | 16 +- .../catalog/io/mapper/FileIOConfigMapper.java | 2 +- .../catalog/io/mapper/GCSIOConfigMapper.java | 30 ++ .../catalog/mapper/CatalogConfigMapper.java | 4 +- .../catalog/mapper/HiveConfigMapper.java | 1 + .../catalog/mapper/JdbcCatalogMapper.java | 3 +- .../flow/jvm/iceberg/lock/LockManager.java | 21 ++ .../jvm/iceberg/resource/IcebergResource.java | 57 ++-- .../jvm/iceberg/service/IcebergService.java | 292 ++++++++++++------ .../iceberg/writer/RecordWriterFactory.java | 41 +++ .../flow/jvm/iceberg/writer/TaskWriter.java | 4 - flow-jvm/src/main/resources/application.yaml | 7 +- .../java/io/peerdb/GreetingResourceIT.java | 8 - .../java/io/peerdb/GreetingResourceTest.java | 20 -- flow/connectors/iceberg/iceberg.go | 62 ++-- flow/connectors/iceberg/qrep.go | 80 ++--- flow/connectors/s3/qrep.go | 1 + flow/model/qvalue/avro_converter.go | 52 +++- protos/flow-jvm.proto | 1 + protos/peers.proto | 18 +- 28 files changed, 523 insertions(+), 395 deletions(-) delete mode 100644 flow-jvm/src/main/java/io/peerdb/GreetingResource.java delete mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/ErrorInterceptor.java delete mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/GrpcError.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/GCSIOConfigMapper.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/RecordWriterFactory.java delete mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/TaskWriter.java delete mode 100644 flow-jvm/src/native-test/java/io/peerdb/GreetingResourceIT.java delete mode 100644 flow-jvm/src/test/java/io/peerdb/GreetingResourceTest.java diff --git a/flow-jvm/build.gradle b/flow-jvm/build.gradle index 9ea6bdc5f7..17f88341e1 100644 --- a/flow-jvm/build.gradle +++ b/flow-jvm/build.gradle @@ -9,7 +9,7 @@ repositories { } ext { - icebergLibVersion = '1.4.2' + icebergLibVersion = '1.5.2' } @@ -34,7 +34,13 @@ dependencies { // AWS Dependencies implementation "org.apache.iceberg:iceberg-aws:${icebergLibVersion}" - runtimeOnly 'software.amazon.awssdk:bundle:2.25.60' + runtimeOnly "org.apache.iceberg:iceberg-aws-bundle:${icebergLibVersion}" +// runtimeOnly 'software.amazon.awssdk:bundle:2.25.60' + + // GCP Dependencies + implementation "org.apache.iceberg:iceberg-gcp:${icebergLibVersion}" + // This is currently causing issues with GRPC versions mismatch +// runtimeOnly "org.apache.iceberg:iceberg-gcp-bundle:${icebergLibVersion}" testImplementation 'io.quarkus:quarkus-junit5' diff --git a/flow-jvm/gradle.properties b/flow-jvm/gradle.properties index dde45ebcf2..d87a1b7a1f 100644 --- a/flow-jvm/gradle.properties +++ b/flow-jvm/gradle.properties @@ -2,6 +2,6 @@ #Fri May 24 04:59:17 IST 2024 quarkusPlatformArtifactId=quarkus-bom quarkusPlatformGroupId=io.quarkus.platform -quarkusPlatformVersion=3.10.2 +quarkusPlatformVersion=3.11.1 quarkusPluginId=io.quarkus -quarkusPluginVersion=3.10.2 +quarkusPluginVersion=3.11.1 diff --git a/flow-jvm/src/main/java/io/peerdb/GreetingResource.java b/flow-jvm/src/main/java/io/peerdb/GreetingResource.java deleted file mode 100644 index 3a8db22844..0000000000 --- a/flow-jvm/src/main/java/io/peerdb/GreetingResource.java +++ /dev/null @@ -1,16 +0,0 @@ -package io.peerdb; - -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; - -@Path("/hello") -public class GreetingResource { - - @GET - @Produces(MediaType.TEXT_PLAIN) - public String hello() { - return "Hello from Quarkus REST"; - } -} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java index a263ca80f1..98894fa438 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java @@ -2,6 +2,7 @@ import io.grpc.Metadata; import io.grpc.ServerCall; +import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.quarkus.grpc.ExceptionHandler; import io.quarkus.grpc.ExceptionHandlerProvider; @@ -9,24 +10,28 @@ import jakarta.enterprise.context.ApplicationScoped; @ApplicationScoped -public class DefaultExceptionHandlerProvider implements ExceptionHandlerProvider{ +public class DefaultExceptionHandlerProvider implements ExceptionHandlerProvider { public static boolean invoked; + private static Exception toStatusException(Throwable t) { + return Status.fromThrowable(t).withDescription(t.getMessage()).asRuntimeException(); + } + @Override public ExceptionHandler createHandler(ServerCall.Listener listener, ServerCall serverCall, Metadata metadata) { - return new HelloExceptionHandler<>(listener, serverCall, metadata); + return new DefaultExceptionHandler<>(listener, serverCall, metadata); } @Override public Throwable transform(Throwable t) { invoked = true; - Log.errorf(t, "Transforming exception"); - return ExceptionHandlerProvider.toStatusException(t, true); + Log.errorf(t, "Received error in gRPC call"); + return toStatusException(t); } - private static class HelloExceptionHandler extends ExceptionHandler { - public HelloExceptionHandler(ServerCall.Listener listener, ServerCall call, Metadata metadata) { + private static class DefaultExceptionHandler extends ExceptionHandler { + public DefaultExceptionHandler(ServerCall.Listener listener, ServerCall call, Metadata metadata) { super(listener, call, metadata); } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/ErrorInterceptor.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/ErrorInterceptor.java deleted file mode 100644 index 17a4068975..0000000000 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/ErrorInterceptor.java +++ /dev/null @@ -1,37 +0,0 @@ -package io.peerdb.flow.jvm; - - -import io.grpc.*; -import io.quarkus.grpc.GlobalInterceptor; -import io.quarkus.logging.Log; -import jakarta.enterprise.context.ApplicationScoped; - -@ApplicationScoped -@GlobalInterceptor -public class ErrorInterceptor implements ServerInterceptor { - @Override - public ServerCall.Listener interceptCall(ServerCall serverCall, Metadata metadata, ServerCallHandler serverCallHandler) { - Log.infof("Intercepting call", metadata); - System.out.println("Intercepting call sout"); - - ServerCall listener = new ForwardingServerCall.SimpleForwardingServerCall(serverCall) { - @Override - public void sendMessage(RespT message) { - Log.debugf("Sending message for method: %s: {%s}", serverCall.getMethodDescriptor().getFullMethodName(), message); - super.sendMessage(message); - } - }; - - return new ForwardingServerCallListener.SimpleForwardingServerCallListener(serverCallHandler.startCall(listener, metadata)) { - - @Override - public void onMessage(ReqT message) { - Log.debugf("Received message from client for method: %s: {%s}", serverCall.getMethodDescriptor().getFullMethodName(), message); - super.onMessage(message); - } - - }; - -// return serverCallHandler.startCall(serverCall, metadata); - } -} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/GrpcError.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/GrpcError.java deleted file mode 100644 index 812a5af68a..0000000000 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/GrpcError.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.peerdb.flow.jvm; - -public class GrpcError extends RuntimeException { - public GrpcError(String message) { - super(message); - } - - public GrpcError(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java new file mode 100644 index 0000000000..28853557e1 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java @@ -0,0 +1,49 @@ +package io.peerdb.flow.jvm; + + +import com.google.common.base.Stopwatch; +import io.grpc.*; +import io.quarkus.grpc.GlobalInterceptor; +import io.quarkus.logging.Log; +import jakarta.enterprise.context.ApplicationScoped; + +import java.util.concurrent.TimeUnit; + +@ApplicationScoped +@GlobalInterceptor +public class RequestLoggingInterceptor implements ServerInterceptor { + @Override + public ServerCall.Listener interceptCall(ServerCall serverCall, Metadata metadata, ServerCallHandler serverCallHandler) { + var stopwatch = Stopwatch.createStarted(); + Log.infof("Received request for method {%s}", serverCall.getMethodDescriptor().getFullMethodName()); + ServerCall listener = new ForwardingServerCall.SimpleForwardingServerCall<>(serverCall) { + }; + return new CallListener<>(serverCallHandler, listener, metadata, serverCall, stopwatch); + + } + + private static class CallListener extends ForwardingServerCallListener.SimpleForwardingServerCallListener { + + private final ServerCall serverCall; + private final Stopwatch stopwatch; + + public CallListener(ServerCallHandler serverCallHandler, ServerCall listener, Metadata metadata, ServerCall serverCall, Stopwatch stopwatch) { + super(serverCallHandler.startCall(listener, metadata)); + this.serverCall = serverCall; + this.stopwatch = stopwatch; + } + + @Override + public void onMessage(ReqT message) { + Log.debugf("Received request for method: %s: {%s}", serverCall.getMethodDescriptor().getFullMethodName(), message); + super.onMessage(message); + } + + @Override + public void onComplete() { + Log.infof("Call completed for method: {%s} in %d ms", serverCall.getMethodDescriptor().getFullMethodName(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); + super.onComplete(); + } + + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java index 037f11ccab..e559924f33 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java @@ -1,61 +1,43 @@ package io.peerdb.flow.jvm.iceberg.avro; -import io.quarkus.logging.Log; import org.apache.avro.Schema; import org.apache.avro.file.SeekableByteArrayInput; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.DecoderFactory; -import org.apache.iceberg.data.Record; +import org.apache.iceberg.avro.AvroSchemaUtil; +import org.apache.iceberg.data.avro.DataReader; import java.io.IOException; public class AvroIcebergConverter { private final org.apache.iceberg.Schema icebergSchema; - private final Schema avroSchema; + private final Schema icebergAvroSchema; + private final DataReader dataReader; - public AvroIcebergConverter(String avroSchemaString, org.apache.iceberg.Schema icebergSchema) { + public AvroIcebergConverter(String avroSchemaString, org.apache.iceberg.Schema icebergSchema, String tableName) { + this(new Schema.Parser().parse(avroSchemaString), icebergSchema, tableName); - this.icebergSchema = icebergSchema; - var avroSchemaParser = new org.apache.avro.Schema.Parser(); - this.avroSchema = avroSchemaParser.parse(avroSchemaString); } - public AvroIcebergConverter(Schema avroSchema, org.apache.iceberg.Schema icebergSchema) { + public AvroIcebergConverter(Schema sourceAvroSchema, org.apache.iceberg.Schema icebergSchema, String tableName) { this.icebergSchema = icebergSchema; - this.avroSchema = avroSchema; - } + this.icebergAvroSchema = AvroSchemaUtil.convert(icebergSchema, tableName); + this.dataReader = DataReader.create(icebergSchema, icebergAvroSchema); + this.dataReader.setSchema(sourceAvroSchema); - public GenericRecord toAvroRecord(byte[] bytes) throws IOException { - var reader = new GenericDatumReader(avroSchema); - try (var byteStream = new SeekableByteArrayInput(bytes)) { - // The below code is for avro binary data files -// var dataFileReader = DataFileReader.openReader(byteStream, reader); -// if (!dataFileReader.hasNext()) { -// Log.errorf("No records found!"); -// return; -// } -// var record = dataFileReader.next(); + } + public org.apache.iceberg.data.GenericRecord toIcebergRecord(byte[] avroBytes) throws IOException { + try (var byteStream = new SeekableByteArrayInput(avroBytes)) { var binaryDecoder = DecoderFactory.get().binaryDecoder(byteStream, null); - return reader.read(null, binaryDecoder); + return this.dataReader.read(null, binaryDecoder); } } - - public org.apache.iceberg.data.GenericRecord toIcebergRecord(GenericRecord avroRecord) { - var genericRecord = org.apache.iceberg.data.GenericRecord.create(icebergSchema); - for (var field : icebergSchema.columns()) { - var fieldName = field.name(); - var fieldValue = avroRecord.get(fieldName); - Log.tracef("Will set Field: %s, Value: %s, Current Record: %s", fieldName, fieldValue, genericRecord); - genericRecord.setField(fieldName, fieldValue); - } - return genericRecord; + public org.apache.iceberg.Schema getIcebergSchema() { + return icebergSchema; } - public org.apache.iceberg.data.GenericRecord toIcebergRecord(byte[] bytes) throws IOException { - return toIcebergRecord(toAvroRecord(bytes)); + public Schema getIcebergAvroSchema() { + return icebergAvroSchema; } - } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java index 6346a40127..b6da3596f5 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java @@ -1,6 +1,7 @@ package io.peerdb.flow.jvm.iceberg.catalog; +import io.peerdb.flow.jvm.iceberg.catalog.io.mapper.GCSIOConfigMapper; import io.peerdb.flow.jvm.iceberg.catalog.io.mapper.S3IOConfigMapper; import io.peerdb.flow.jvm.iceberg.catalog.mapper.HiveConfigMapper; import io.peerdb.flow.jvm.iceberg.catalog.mapper.JdbcCatalogMapper; @@ -8,7 +9,6 @@ import io.quarkus.logging.Log; import jakarta.inject.Inject; import jakarta.inject.Singleton; -import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.catalog.Catalog; @@ -26,12 +26,15 @@ public class CatalogLoader { @Inject S3IOConfigMapper s3IOConfigMapper; + @Inject + GCSIOConfigMapper gcsIOConfigMapper; public Catalog loadCatalog(IcebergCatalog icebergCatalogConfig) { var icebergIOConfig = icebergCatalogConfig.getIoConfig(); var fileIoConfig = switch (icebergIOConfig.getConfigCase()) { case S3 -> s3IOConfigMapper.map(icebergIOConfig.getS3()); + case GCS -> gcsIOConfigMapper.map(icebergIOConfig.getGcs()); default -> { Log.errorf("Unexpected value for file io config: %s", icebergIOConfig.getConfigCase()); yield Collections.emptyMap(); @@ -39,11 +42,14 @@ public Catalog loadCatalog(IcebergCatalog icebergCatalogConfig) { }; var catalogconfig = switch (icebergCatalogConfig.getConfigCase()) { - case HIVE -> hiveConfigMapper.map(icebergCatalogConfig.getCommonConfig(), icebergCatalogConfig.getHive(), fileIoConfig); - case JDBC -> jdbcCatalogMapper.map(icebergCatalogConfig.getCommonConfig(), icebergCatalogConfig.getJdbc(), fileIoConfig); - default -> throw new IllegalStateException("Unexpected value for catalog config: " + icebergCatalogConfig.getConfigCase()); + case HIVE -> + hiveConfigMapper.map(icebergCatalogConfig.getCommonConfig(), icebergCatalogConfig.getHive(), fileIoConfig); + case JDBC -> + jdbcCatalogMapper.map(icebergCatalogConfig.getCommonConfig(), icebergCatalogConfig.getJdbc(), fileIoConfig); + default -> + throw new IllegalStateException("Unexpected value for catalog config: " + icebergCatalogConfig.getConfigCase()); }; // TODO look at hadoop - return CatalogUtil.buildIcebergCatalog(catalogconfig.get(CatalogProperties.CATALOG_IMPL), catalogconfig, null); + return CatalogUtil.buildIcebergCatalog(icebergCatalogConfig.getCommonConfig().getName(), catalogconfig, null); } } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FileIOConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FileIOConfigMapper.java index de64c10d27..c62acc2889 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FileIOConfigMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FileIOConfigMapper.java @@ -20,7 +20,7 @@ protected Map mapCommon() { public final Map map(T config) { var map = new HashMap<>(mapCommon()); map.putAll(this.mapSpecific(config)); - Log.infof("Mapped IO config: %s", map); + Log.debugf("Mapped IO config: %s", map); return map; } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/GCSIOConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/GCSIOConfigMapper.java new file mode 100644 index 0000000000..3b9c836ddb --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/GCSIOConfigMapper.java @@ -0,0 +1,30 @@ +package io.peerdb.flow.jvm.iceberg.catalog.io.mapper; + +import com.google.common.collect.ImmutableMap; +import io.peerdb.flow.peers.IcebergGCSIoConfig; +import jakarta.enterprise.context.ApplicationScoped; + +import java.util.Map; + +@ApplicationScoped +public class GCSIOConfigMapper extends FileIOConfigMapper { + @Override + protected Map mapSpecific(IcebergGCSIoConfig config) { + // TODO complete this + var builder = ImmutableMap.builder(); +// .put(GCPProperties.GCS_PROJECT_ID, config.getAccessKeyId()) +// .put(S3FileIOProperties.SECRET_ACCESS_KEY, config.getSecretAccessKey()); +// if (config.hasEndpoint()) { +// builder.put(S3FileIOProperties.ENDPOINT, config.getEndpoint()); +// } +// if (config.hasPathStyleAccess()) { +// builder.put(S3FileIOProperties.PATH_STYLE_ACCESS, config.getPathStyleAccess()); +// } + return builder.build(); + } + + @Override + public String implementationClass() { + return "org.apache.iceberg.gcp.gcs.GCSFileIO"; + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java index 709e6d9764..a28b399334 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java @@ -19,7 +19,7 @@ protected Map mapCommon(CommonIcebergCatalog config) { builder.put(CatalogProperties.CLIENT_POOL_SIZE, String.valueOf(config.getClientPoolSize())); } if (config.hasCacheEnabled()) { - builder.put(CatalogProperties.CACHE_ENABLED, String.valueOf(config.getCacheEnabled())); + builder.put(CatalogProperties.CACHE_ENABLED, String.valueOf(config.getCacheEnabled())); } builder.putAll(config.getAdditionalPropertiesMap()); return builder.build(); @@ -29,7 +29,7 @@ public Map map(CommonIcebergCatalog commonConfig, T config, Map< var map = new HashMap<>(mapCommon(commonConfig)); map.putAll(this.mapSpecific(config)); map.putAll(fileIoConfig); - Log.infof("Mapped catalog config: %s", map); + Log.debugf("Mapped catalog config: %s", map); return map; } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java index c567425387..ba37344df3 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java @@ -11,6 +11,7 @@ public class HiveConfigMapper extends CatalogConfigMapper { @Override protected Map mapSpecific(HiveIcebergCatalog config) { + // TODO complete this return Collections.emptyMap(); } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/JdbcCatalogMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/JdbcCatalogMapper.java index a1adc4c8f6..9e759a7527 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/JdbcCatalogMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/JdbcCatalogMapper.java @@ -4,7 +4,6 @@ import jakarta.inject.Singleton; import org.apache.iceberg.jdbc.JdbcCatalog; -import java.util.Collections; import java.util.Map; import java.util.stream.Collectors; @@ -17,7 +16,7 @@ protected Map mapSpecific(JdbcIcebergCatalog config) { return Map.of( "user", config.getUser(), "password", config.getPassword(), - "useSSL", config.hasUseSsl() ? String.valueOf(config.getUseSsl()) : "true", + "useSSL", config.hasUseSsl() ? String.valueOf(config.getUseSsl()) : "true", "verifyServerCertificate", config.hasVerifyServerCertificate() ? String.valueOf(config.getVerifyServerCertificate()) : "false" ).entrySet().stream().collect(Collectors.toMap(e -> JdbcCatalog.PROPERTY_PREFIX + e.getKey(), Map.Entry::getValue)); diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java new file mode 100644 index 0000000000..bae324d20d --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java @@ -0,0 +1,21 @@ +package io.peerdb.flow.jvm.iceberg.lock; + + +import com.google.common.util.concurrent.Striped; +import jakarta.enterprise.context.Dependent; + +import java.util.concurrent.locks.Lock; + +@Dependent +public class LockManager { + int stripeCount = 10_000; + + Striped locker = Striped.lock(stripeCount); + + + // This is just abstracted out to enable changing the lock implementation in the future + public Lock newLock(Object key) { + return locker.get(key); + } + +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java index 604390995a..30b8cc11f1 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java @@ -1,76 +1,53 @@ package io.peerdb.flow.jvm.iceberg.resource; -import com.google.common.collect.Streams; import io.peerdb.flow.jvm.grpc.*; -import io.peerdb.flow.jvm.iceberg.catalog.CatalogLoader; import io.peerdb.flow.jvm.iceberg.service.IcebergService; import io.quarkus.grpc.GrpcService; -import io.quarkus.logging.Log; +import io.smallrye.common.annotation.Blocking; import io.smallrye.common.annotation.RunOnVirtualThread; import io.smallrye.mutiny.Uni; import jakarta.inject.Inject; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.data.IcebergGenerics; -import java.io.IOException; import java.util.Optional; @GrpcService public class IcebergResource implements IcebergProxyService { - @Inject - CatalogLoader catalogLoader; - @Inject IcebergService icebergService; @RunOnVirtualThread @Override public Uni createTable(CreateTableRequest request) { - var tableInfo = request.getTableInfo(); - var icebergCatalog = tableInfo.getIcebergCatalog(); - - var catalog = catalogLoader.loadCatalog(icebergCatalog); - var icebergSchema = icebergService.getIcebergSchema(request.getSchema()); - var table = catalog.createTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName()), icebergSchema); - return Uni.createFrom().item(CreateTableResponse.newBuilder().setTableName(table.name()).build()); + return Uni.createFrom().item(() -> + CreateTableResponse.newBuilder() + .setTableName( + icebergService.createTable(request.getTableInfo(), request.getSchema()).name() + ).build()); } @RunOnVirtualThread @Override public Uni dropTable(DropTableRequest request) { - var tableInfo = request.getTableInfo(); - var icebergCatalog = tableInfo.getIcebergCatalog(); - var catalog = catalogLoader.loadCatalog(icebergCatalog); - var droppedTable = catalog.dropTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName()), request.getPurge()); - return Uni.createFrom().item(DropTableResponse.newBuilder().setSuccess(droppedTable).build()); + return Uni.createFrom().item(() -> DropTableResponse.newBuilder().setSuccess(icebergService.dropTable(request.getTableInfo(), request.getPurge())).build()); } + @RunOnVirtualThread @Override public Uni countRecords(CountRecordRequest request) { - var tableInfo = request.getTableInfo(); - var icebergCatalog = tableInfo.getIcebergCatalog(); - var catalog = catalogLoader.loadCatalog(icebergCatalog); - var table = catalog.loadTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName())); - - Log.debugf("For table %s, schema is %s", tableInfo.getTableName(), table.schema()); - var count = 0L; - try (var tableScan = IcebergGenerics.read(table).build()) { - count = Streams.stream(tableScan.iterator()).reduce(0L, (current, record) -> current + 1L, Long::sum); - } catch (IOException e) { - Log.errorf(e, "Error reading table %s", tableInfo.getTableName()); - throw new RuntimeException(e); - } - return Uni.createFrom().item(CountRecordResponse.newBuilder().setCount(count).build()); + return Uni.createFrom().item(() -> { + var count = icebergService.processTableCountRequest(request); + return CountRecordResponse.newBuilder().setCount(count).build(); + }); } + @RunOnVirtualThread @Override public Uni insertChanges(InsertChangesRequest request) { return Uni.createFrom() - .item(InsertChangesResponse.newBuilder() + .item(() -> InsertChangesResponse.newBuilder() .setSuccess( icebergService.insertChanges( request.getTableInfo(), @@ -82,8 +59,12 @@ public Uni insertChanges(InsertChangesRequest request) { } + @Blocking @Override public Uni appendRecords(AppendRecordsRequest request) { - return Uni.createFrom().item(AppendRecordsResponse.newBuilder().setSuccess(icebergService.appendRecords(request.getTableInfo(), request.getSchema(), request.getRecordsList())).build()); + return Uni.createFrom().item(() -> AppendRecordsResponse.newBuilder() + .setSuccess( + icebergService.processAppendRecordsRequest(request)) + .build()); } } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index 94851007a4..7ab4911631 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -1,42 +1,210 @@ package io.peerdb.flow.jvm.iceberg.service; -import io.peerdb.flow.jvm.grpc.BranchOptions; -import io.peerdb.flow.jvm.grpc.InsertRecord; -import io.peerdb.flow.jvm.grpc.RecordChange; -import io.peerdb.flow.jvm.grpc.TableInfo; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Streams; +import io.peerdb.flow.jvm.grpc.*; import io.peerdb.flow.jvm.iceberg.avro.AvroIcebergConverter; import io.peerdb.flow.jvm.iceberg.catalog.CatalogLoader; +import io.peerdb.flow.jvm.iceberg.lock.LockManager; +import io.peerdb.flow.jvm.iceberg.writer.RecordWriterFactory; import io.quarkus.logging.Log; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; import org.apache.iceberg.Schema; -import org.apache.iceberg.TableProperties; +import org.apache.iceberg.Table; import org.apache.iceberg.avro.AvroSchemaUtil; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.data.GenericAppenderFactory; -import org.apache.iceberg.io.OutputFileFactory; -import org.apache.iceberg.io.UnpartitionedWriter; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.io.TaskWriter; import org.apache.iceberg.io.WriteResult; -import org.apache.iceberg.util.PropertyUtil; import java.io.IOException; -import java.util.*; - -import static org.apache.iceberg.TableProperties.WRITE_TARGET_FILE_SIZE_BYTES; -import static org.apache.iceberg.TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT; +import java.io.UncheckedIOException; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; @ApplicationScoped public class IcebergService { + static final int maxIdempotencyKeyAgeDays = 7; @Inject CatalogLoader catalogLoader; + @Inject + LockManager lockManager; + @Inject + RecordWriterFactory recordWriterFactory; + private static void writeRecordStream(Stream recordStream, AvroIcebergConverter converter, TaskWriter writer) { + recordStream.parallel().map(insertRecord -> { + try { + return converter.toIcebergRecord(insertRecord.getRecord().toByteArray()); + } catch (IOException e) { + Log.errorf(e, "Error while converting record"); + throw new UncheckedIOException(e); + } + }).toList().forEach(record -> { + try { + writer.write(record); + } catch (IOException e) { + Log.errorf(e, "Error while writing record"); + throw new UncheckedIOException(e); + } + }); + } + + private Object getTableLockKey(TableInfo tableInfo) { + return List.of(tableInfo.getIcebergCatalog(), tableInfo.getNamespaceList(), tableInfo.getTableName()); + } + + private TableIdentifier getTableIdentifier(TableInfo tableInfo) { + return TableIdentifier.parse(tableInfo.getTableName()); + } + + public Table createTable(TableInfo tableInfo, String schema) { + var icebergCatalog = tableInfo.getIcebergCatalog(); + + var catalog = catalogLoader.loadCatalog(icebergCatalog); + var typeSchema = getIcebergSchema(schema); + // TODO Below require that the primary keys are non-null +// var fieldList = typeSchema.columns(); +// var primaryKeyFieldIds = request.getTableInfo().getPrimaryKeyList().stream().map(pk -> +// Objects.requireNonNull(typeSchema.findField(pk), String.format("Primary key %s not found in schema", pk)).fieldId() +// ).collect(Collectors.toSet()); +// var icebergSchema = new Schema(fieldList, primaryKeyFieldIds); + var icebergSchema = typeSchema; + Preconditions.checkArgument(icebergSchema.asStruct().equals(typeSchema.asStruct()), "Primary key based schema not equivalent to type schema [%s!=%s]", icebergSchema.asStruct(), typeSchema.asStruct()); + Log.infof("Will now create table %s", tableInfo.getTableName()); + var table = catalog.createTable(getTableIdentifier(tableInfo), icebergSchema); + Log.infof("Created table %s", tableInfo.getTableName()); + return table; + } + + public boolean dropTable(TableInfo tableInfo, boolean purge) { + var icebergCatalog = tableInfo.getIcebergCatalog(); + var catalog = catalogLoader.loadCatalog(icebergCatalog); + return catalog.dropTable(getTableIdentifier(tableInfo), purge); + } + + public boolean processAppendRecordsRequest(AppendRecordsRequest request) { + return appendRecords(request.getTableInfo(), + request.getSchema(), + request.getRecordsList(), + Optional.ofNullable(request.hasIdempotencyKey() ? request.getIdempotencyKey() : null)); + } + + private boolean appendRecords(TableInfo tableInfo, String avroSchema, List insertRecords, Optional idempotencyKey) { + var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); + var table = icebergCatalog.loadTable(getTableIdentifier(tableInfo)); + + if (isAppendAlreadyDone(table, idempotencyKey)) { + return true; + } + var recordStream = insertRecords.stream(); + var dataFiles = getAppendDataFiles(avroSchema, table, recordStream); + Log.infof("Completed writing %d records for table %s", Arrays.stream(dataFiles).map(ContentFile::recordCount).reduce(0L, Long::sum), table.name()); + + + var lockKey = List.of(tableInfo.getIcebergCatalog().toString(), tableInfo.getNamespaceList(), tableInfo.getTableName()); + Log.infof("Will now acquire lock for table %s by idempotency key %s for lockHashCode: %d", table.name(), idempotencyKey.orElse(""), lockKey.hashCode()); + var lock = lockManager.newLock(lockKey); + lock.lock(); + try { + Log.infof("Acquired lock for table %s by idempotency key %s", table.name(), idempotencyKey.orElse("")); + Log.infof("Will now refresh table %s", table.name()); + table.refresh(); + if (isAppendAlreadyDone(table, idempotencyKey)) { + return true; + } + var transaction = table.newTransaction(); + Log.infof("Will now append files to table %s", table.name()); + var appendFiles = transaction.newAppend(); + + Arrays.stream(dataFiles).forEach(appendFiles::appendFile); + Log.infof("Appended files to table %s", table.name()); + appendFiles.commit(); + Log.infof("Committed files to table %s", table.name()); + idempotencyKey.ifPresent(key -> { + Log.infof("Will now create branch %s for table %s", key, table.name()); + transaction.manageSnapshots().createBranch(getBranchNameFromIdempotencyKey(key)) + .setMaxRefAgeMs(getBranchNameFromIdempotencyKey(key), Duration.ofDays(maxIdempotencyKeyAgeDays).toMillis()) + .commit(); + Log.infof("Created branch %s for table %s", key, table.name()); + }); + transaction.table().refresh(); + + Log.infof("Will now commit transaction for table %s", table.name()); + transaction.commitTransaction(); + Log.infof("Committed transaction for table %s", table.name()); + + return true; + } finally { + lock.unlock(); + Log.infof("Released lock for table %s by idempotency key %s", table.name(), idempotencyKey.get()); + } + + } + + private DataFile[] getAppendDataFiles(String avroSchema, Table table, Stream recordStream) { + WriteResult writeResult; + try (var writer = recordWriterFactory.createRecordWriter(table)) { + var converter = new AvroIcebergConverter(avroSchema, table.schema(), table.name()); + Log.infof("Will now write records to append to table %s", table.name()); + var stopwatch = Stopwatch.createStarted(); + writeRecordStream(recordStream, converter, writer); + Log.infof("Completed writing records to append to table %s in %d ms", table.name(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); + try { + writeResult = writer.complete(); + } catch (IOException e) { + Log.errorf(e, "Error while completing writing records"); + throw new UncheckedIOException(e); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + var dataFiles = writeResult.dataFiles(); + return dataFiles; + } + + private boolean isAppendAlreadyDone(Table table, Optional idempotencyKey) { + if (idempotencyKey.isPresent()) { + var branchName = getBranchNameFromIdempotencyKey(idempotencyKey.get()); + if (table.refs().containsKey(branchName)) { + Log.warnf("Already committed work found for table %s with idempotency key %s", table.name(), idempotencyKey.get()); + return true; + } + } + return false; + } + + public long processTableCountRequest(CountRecordRequest request) { + var tableInfo = request.getTableInfo(); + var icebergCatalog = tableInfo.getIcebergCatalog(); + var catalog = catalogLoader.loadCatalog(icebergCatalog); + var table = catalog.loadTable(getTableIdentifier(tableInfo)); + + Log.debugf("For table %s, schema is %s", tableInfo.getTableName(), table.schema()); + var count = 0L; + try (var tableScan = IcebergGenerics.read(table).build()) { + count = Streams.stream(tableScan.iterator()).reduce(0L, (current, record) -> current + 1L, Long::sum); + } catch (IOException e) { + Log.errorf(e, "Error reading table %s", tableInfo.getTableName()); + throw new RuntimeException(e); + } + return count; + } public boolean insertChanges(TableInfo tableInfo, String avroSchema, List recordChanges, Optional branchOptions) { + // TODO this is for CDC, will be done later var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); - var table = icebergCatalog.loadTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName())); + var table = icebergCatalog.loadTable(getTableIdentifier(tableInfo)); if (branchOptions.isPresent()) { var branchName = branchOptions.get().getBranch(); if (table.refs().containsKey(branchName)) { @@ -47,21 +215,14 @@ public boolean insertChanges(TableInfo tableInfo, String avroSchema, List table.newTransaction().manageSnapshots().removeBranch(branchName).commit(); - default -> throw new IllegalArgumentException(String.format("Unrecognized branch create conflict policy %s", branchOptions.get().getBranchCreateConflictPolicy())); + default -> + throw new IllegalArgumentException(String.format("Unrecognized branch create conflict policy %s", branchOptions.get().getBranchCreateConflictPolicy())); } } } - var appenderFactory = new GenericAppenderFactory(table.schema(), table.spec()); - var format = FileFormat.fromString(table.properties().getOrDefault(TableProperties.DEFAULT_FILE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)); - var outputFileFactory = OutputFileFactory.builderFor(table, 1, System.currentTimeMillis()) - .defaultSpec(table.spec()) - .operationId(UUID.randomUUID().toString()) - .format(format) - .build(); - var targetFileSize = PropertyUtil.propertyAsLong(table.properties(), WRITE_TARGET_FILE_SIZE_BYTES, WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT); - - var converter = new AvroIcebergConverter(avroSchema, table.schema()); - var writer = new UnpartitionedWriter<>(table.spec(), format, appenderFactory, outputFileFactory, table.io(), targetFileSize); + var writer = recordWriterFactory.createRecordWriter(table); + + var converter = new AvroIcebergConverter(avroSchema, table.schema(), table.name()); recordChanges.forEach(recordChange -> { switch (recordChange.getChangeCase()) { case INSERT: @@ -84,21 +245,9 @@ public boolean insertChanges(TableInfo tableInfo, String avroSchema, List insertRecords) { - var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); - var table = icebergCatalog.loadTable(TableIdentifier.of(Namespace.of(tableInfo.getNamespaceList().toArray(String[]::new)), tableInfo.getTableName())); - - // TODO add support for identifier fields - var appenderFactory = new GenericAppenderFactory(table.schema(), table.spec()); - var format = FileFormat.fromString(table.properties().getOrDefault(TableProperties.DEFAULT_FILE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)); - // TODO add partition ID - var outputFileFactory = OutputFileFactory.builderFor(table, 1, System.currentTimeMillis()) - .defaultSpec(table.spec()) - .operationId(UUID.randomUUID().toString()) - .format(format) - .build(); - var targetFileSize = PropertyUtil.propertyAsLong(table.properties(), WRITE_TARGET_FILE_SIZE_BYTES, WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT); - - var writer = new UnpartitionedWriter<>(table.spec(), format, appenderFactory, outputFileFactory, table.io(), targetFileSize); - var converter = new AvroIcebergConverter(avroSchema, table.schema()); - // TODO can make this parallel for speed up - insertRecords.forEach(insertRecord -> { - try { - var encoded = Base64.getEncoder().encodeToString(insertRecord.getRecord().toByteArray()); - Log.tracef("Received record: %s", encoded); - var genericRecord = converter.toIcebergRecord(insertRecord.getRecord().toByteArray()); - Log.tracef("Writing record: %s", genericRecord); - writer.write(genericRecord); - } catch (IOException e) { - Log.errorf(e, "Error while converting/writing record"); - throw new RuntimeException(e); - } - }); - WriteResult writeResult; - try { - writeResult = writer.complete(); - } catch (IOException e) { - Log.errorf(e, "Error while completing writing records"); - throw new RuntimeException(e); - } - - var branchName = "_flow_something_something_temp"; - - if (table.refs().containsKey(branchName)) { - // Maybe skip everything and just return (depending on input) to skip to normalize - // Or maybe we should drop and redo everything (as there might be some dirty tables) - } - - var transaction = table.newTransaction(); -// transaction.manageSnapshots().createBranch(branchName) -//// .setMaxRefAgeMs() -//// .setMinSnapshotsToKeep() -//// .setMaxSnapshotAgeMs() -// .commit();; - - - var appendFiles = transaction.newAppend() -// .toBranch(branchName) - ; - - Arrays.stream(writeResult.dataFiles()).forEach(appendFiles::appendFile); - appendFiles.commit(); - transaction.commitTransaction(); - return true; + private String getBranchNameFromIdempotencyKey(String idempotencyKey) { + return String.format("__peerdb-idem-%s", idempotencyKey); } } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/RecordWriterFactory.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/RecordWriterFactory.java new file mode 100644 index 0000000000..1e6ba8d664 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/RecordWriterFactory.java @@ -0,0 +1,41 @@ +package io.peerdb.flow.jvm.iceberg.writer; + + +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.data.GenericAppenderFactory; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.io.OutputFileFactory; +import org.apache.iceberg.io.TaskWriter; +import org.apache.iceberg.io.UnpartitionedWriter; +import org.apache.iceberg.util.PropertyUtil; + +import java.util.UUID; + +import static org.apache.iceberg.TableProperties.WRITE_TARGET_FILE_SIZE_BYTES; +import static org.apache.iceberg.TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT; + +@ApplicationScoped +public class RecordWriterFactory { + public TaskWriter createRecordWriter(Table table) { + // TODO add support for partition ID + return createUnpartitionedRecordWriter(table); + } + + + private TaskWriter createUnpartitionedRecordWriter(Table table) { + var appenderFactory = new GenericAppenderFactory(table.schema(), table.spec()); + var format = FileFormat.fromString(table.properties().getOrDefault(TableProperties.DEFAULT_FILE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)); + var outputFileFactory = OutputFileFactory.builderFor(table, 1, System.currentTimeMillis()) + .defaultSpec(table.spec()) + .operationId(UUID.randomUUID().toString()) + .format(format) + .build(); + var targetFileSize = PropertyUtil.propertyAsLong(table.properties(), WRITE_TARGET_FILE_SIZE_BYTES, WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT); + + return new UnpartitionedWriter<>(table.spec(), format, appenderFactory, outputFileFactory, table.io(), targetFileSize); + } + +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/TaskWriter.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/TaskWriter.java deleted file mode 100644 index 596b622dc3..0000000000 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/TaskWriter.java +++ /dev/null @@ -1,4 +0,0 @@ -//package io.peerdb.flow.jvm.iceberg.writer; -// -//public class UnpartitionedTaskWriter extends { -//} diff --git a/flow-jvm/src/main/resources/application.yaml b/flow-jvm/src/main/resources/application.yaml index 5b95921eef..41d67118b8 100644 --- a/flow-jvm/src/main/resources/application.yaml +++ b/flow-jvm/src/main/resources/application.yaml @@ -4,10 +4,12 @@ quarkus: scan-for-imports: all grpc: server: - port: ${FLOW_JVM_GRPC_PORT:9801} + port: ${FLOW_JVM_PORT:9801} # 512 MB max-inbound-message-size: 536870912 + + use-separate-server: false log: level: INFO category: @@ -17,8 +19,9 @@ quarkus: console: level: ${PEERDB_LOG_LEVEL:TRACE} http: - port: ${FLOW_JVM_HTTP_PORT:9802} + port: ${FLOW_JVM_PORT:9801} test: continuous-testing: enabled: disabled + diff --git a/flow-jvm/src/native-test/java/io/peerdb/GreetingResourceIT.java b/flow-jvm/src/native-test/java/io/peerdb/GreetingResourceIT.java deleted file mode 100644 index e5d0418077..0000000000 --- a/flow-jvm/src/native-test/java/io/peerdb/GreetingResourceIT.java +++ /dev/null @@ -1,8 +0,0 @@ -package io.peerdb; - -import io.quarkus.test.junit.QuarkusIntegrationTest; - -@QuarkusIntegrationTest -class GreetingResourceIT extends GreetingResourceTest { - // Execute the same tests but in packaged mode. -} diff --git a/flow-jvm/src/test/java/io/peerdb/GreetingResourceTest.java b/flow-jvm/src/test/java/io/peerdb/GreetingResourceTest.java deleted file mode 100644 index e73353ad0a..0000000000 --- a/flow-jvm/src/test/java/io/peerdb/GreetingResourceTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package io.peerdb; - -import io.quarkus.test.junit.QuarkusTest; -import org.junit.jupiter.api.Test; - -import static io.restassured.RestAssured.given; -import static org.hamcrest.CoreMatchers.is; - -@QuarkusTest -class GreetingResourceTest { - @Test - void testHelloEndpoint() { - given() - .when().get("/hello") - .then() - .statusCode(200) - .body(is("Hello from Quarkus REST")); - } - -} \ No newline at end of file diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go index 9796e20ad2..0ba1714579 100644 --- a/flow/connectors/iceberg/iceberg.go +++ b/flow/connectors/iceberg/iceberg.go @@ -11,7 +11,6 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "log/slog" - "strconv" "time" "go.temporal.io/sdk/log" @@ -92,6 +91,7 @@ func (c *IcebergConnector) PullRecords(ctx context.Context, catalogPool *pgxpool } func (c *IcebergConnector) StartSetupNormalizedTables(ctx context.Context) (any, error) { + // TODO might be better to do all tables in 1 go return nil, nil } @@ -135,17 +135,7 @@ func (c *IcebergConnector) SetupNormalizedTable(ctx context.Context, tx any, tab qFields[i] = qField } - qFields = append(qFields, qvalue.QField{ - Name: softDeleteColName, - Type: qvalue.QValueKindBoolean, - Nullable: true, - }) - - qFields = append(qFields, qvalue.QField{ - Name: syncedAtColName, - Type: qvalue.QValueKindTimestamp, - Nullable: true, - }) + qFields = addPeerMetaColumns(qFields, softDeleteColName, syncedAtColName) avroSchema, err := getAvroSchema(tableIdentifier, qvalue.NewQRecordSchema(qFields)) if err != nil { @@ -174,6 +164,21 @@ func (c *IcebergConnector) SetupNormalizedTable(ctx context.Context, tx any, tab return true, nil } +func addPeerMetaColumns(qFields []qvalue.QField, softDeleteColName string, syncedAtColName string) []qvalue.QField { + qFields = append(qFields, qvalue.QField{ + Name: softDeleteColName, + Type: qvalue.QValueKindBoolean, + Nullable: true, + }) + + qFields = append(qFields, qvalue.QField{ + Name: syncedAtColName, + Type: qvalue.QValueKindTimestampTZ, + Nullable: true, + }) + return qFields +} + func NewIcebergConnector( ctx context.Context, config *protos.IcebergConfig, @@ -202,7 +207,7 @@ func NewIcebergConnector( func (c *IcebergConnector) CreateRawTable(_ context.Context, req *protos.CreateRawTableInput) (*protos.CreateRawTableOutput, error) { // TODO - c.logger.Info("CreateRawTable for S3 is a no-op") + c.logger.Info("CreateRawTable for Iceberg is a no-op") return nil, nil } @@ -212,7 +217,7 @@ func (c *IcebergConnector) Close() error { func (c *IcebergConnector) ValidateCheck(ctx context.Context) error { // Create a table with a random name based on current time - tableName := fmt.Sprintf("peerdb_test_flow_%d", time.Now().Unix()) + tableName := fmt.Sprintf("__peerdb_test_flow_%d", time.Now().Unix()) c.logger.Debug("Will try to create iceberg table", "table", tableName) table, err := c.proxyClient.CreateTable(ctx, &protos.CreateTableRequest{ @@ -257,37 +262,14 @@ func (c *IcebergConnector) ConnectionActive(ctx context.Context) error { func (c *IcebergConnector) SyncRecords(ctx context.Context, req *model.SyncRecordsRequest[model.RecordItems]) (*model.SyncResponse, error) { tableNameRowsMapping := utils.InitialiseTableRowsMap(req.TableMappings) - streamReq := model.NewRecordsToStreamRequest(req.Records.GetRecords(), tableNameRowsMapping, req.SyncBatchID) - records := req.Records.GetRecords() - a := <-records - - items := a.GetItems() - qValue := items.GetColumnValue("asd") - req.TableNameSchemaMapping["asd"].Columns - - if err != nil { - return nil, fmt.Errorf("failed to convert records to raw table stream: %w", err) - } - qrepConfig := &protos.QRepConfig{ - FlowJobName: req.FlowJobName, - DestinationTableIdentifier: "raw_table_" + req.FlowJobName, - } - partition := &protos.QRepPartition{ - PartitionId: strconv.FormatInt(req.SyncBatchID, 10), - } - numRecords, err := c.SyncQRepRecords(ctx, qrepConfig, partition, recordStream) - if err != nil { - return nil, err - } - c.logger.Info(fmt.Sprintf("Synced %d records", numRecords)) lastCheckpoint := req.Records.GetLastCheckpoint() - err = c.FinishBatch(ctx, req.FlowJobName, req.SyncBatchID, lastCheckpoint) + err := c.FinishBatch(ctx, req.FlowJobName, req.SyncBatchID, lastCheckpoint) if err != nil { c.logger.Error("failed to increment id", "error", err) return nil, err } - + numRecords := 0 return &model.SyncResponse{ LastSyncedCheckpointID: lastCheckpoint, NumRecordsSynced: int64(numRecords), @@ -297,6 +279,6 @@ func (c *IcebergConnector) SyncRecords(ctx context.Context, req *model.SyncRecor } func (c *IcebergConnector) ReplayTableSchemaDeltas(_ context.Context, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error { - c.logger.Info("ReplayTableSchemaDeltas for S3 is a no-op") + c.logger.Info("ReplayTableSchemaDeltas for Iceberg is a no-op") return nil } diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go index 34f15b42f6..60f0f766ec 100644 --- a/flow/connectors/iceberg/qrep.go +++ b/flow/connectors/iceberg/qrep.go @@ -6,6 +6,7 @@ import ( "github.com/PeerDB-io/peer-flow/logger" "github.com/linkedin/goavro/v2" "log/slog" + "time" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" @@ -20,6 +21,7 @@ func (c *IcebergConnector) SyncQRepRecords( ) (int, error) { schema := stream.Schema() + schema.Fields = addPeerMetaColumns(schema.Fields, config.SoftDeleteColName, config.SyncedAtColName) dstTableName := config.DestinationTableIdentifier avroSchema, err := getAvroSchema(dstTableName, schema) @@ -39,9 +41,19 @@ func (c *IcebergConnector) SyncQRepRecords( } binaryRecords := make([]*protos.InsertRecord, 0) for record := range stream.Records { + + // Add soft delete + record = append(record, qvalue.QValueBoolean{ + Val: false, + }) + // add synced at colname + record = append(record, qvalue.QValueTimestampTZ{ + Val: time.Now(), + }) + converted, err := avroConverter.Convert(record) if err != nil { - return 0, fmt.Errorf("failed to convert QRecord to Avro-compatible map: %w", err) + return 0, err } binaryData := make([]byte, 0) native, err := codec.BinaryFromNative(binaryData, converted) @@ -54,6 +66,8 @@ func (c *IcebergConnector) SyncQRepRecords( } + requestIdempotencyKey := fmt.Sprintf("_peerdb_qrep-%s-%s", config.FlowJobName, partition.PartitionId) + appendRecordsResponse, err := c.proxyClient.AppendRecords(ctx, &protos.AppendRecordsRequest{ TableInfo: &protos.TableInfo{ @@ -61,9 +75,11 @@ func (c *IcebergConnector) SyncQRepRecords( TableName: dstTableName, IcebergCatalog: c.config.CatalogConfig, //PrimaryKey: nil, + }, - Schema: avroSchema.Schema, - Records: binaryRecords, + Schema: avroSchema.Schema, + Records: binaryRecords, + IdempotencyKey: &requestIdempotencyKey, }, ) @@ -71,34 +87,15 @@ func (c *IcebergConnector) SyncQRepRecords( return 0, err } - logger.LoggerFromCtx(ctx).Info("AppendRecordsResponse", slog.Any("response", appendRecordsResponse)) - - //numRecords, err := c.writeToAvroFile(ctx, stream, avroSchema, partition.PartitionId, config.FlowJobName) - //if err != nil { - // return 0, err - //} + logger.LoggerFromCtx(ctx).Info("AppendRecordsResponse", slog.Any("response", appendRecordsResponse.Success)) + err = c.PostgresMetadata.FinishQRepPartition(ctx, partition, config.FlowJobName, time.Now()) + if err != nil { + return 0, err + } return len(binaryRecords), nil } -//func (c *IcebergConnector) writeToIceberg( -// ctx context.Context, -// stream *model.QRecordStream, -// avroSchema *model.QRecordAvroSchemaDefinition, -// destinationTableName string, -//) (int, error) { -// c.proxyClient.InsertChanges(ctx, &protos.InsertChangesRequest{ -// TableInfo: &protos.TableInfo{ -// Namespace: nil, -// TableName: destinationTableName, -// IcebergCatalog: c.config.CatalogConfig, -// PrimaryKey:, -// }, -// Schema: avroSchema.Schema, -// Changes: nil, -// }) -//} - func getAvroSchema( dstTableName string, schema qvalue.QRecordSchema, @@ -111,30 +108,6 @@ func getAvroSchema( return avroSchema, nil } -//func (c *IcebergConnector) writeToAvroFile( -// ctx context.Context, -// stream *model.QRecordStream, -// avroSchema *model.QRecordAvroSchemaDefinition, -// partitionID string, -// jobName string, -//) (int, error) { -// s3o, err := utils.NewS3BucketAndPrefix(c.url) -// if err != nil { -// return 0, fmt.Errorf("failed to parse bucket path: %w", err) -// } -// -// s3AvroFileKey := fmt.Sprintf("%s/%s/%s.avro", s3o.Prefix, jobName, partitionID) -// -// writer := avro.NewPeerDBOCFWriter(stream, avroSchema, avro.CompressNone, protos.DBType_SNOWFLAKE) -// avroFile, err := writer.WriteRecordsToS3(ctx, s3o.Bucket, s3AvroFileKey, c.credentialsProvider) -// if err != nil { -// return 0, fmt.Errorf("failed to write records to S3: %w", err) -// } -// defer avroFile.Cleanup() -// -// return avroFile.NumRecords, nil -//} - // S3 just sets up destination, not metadata tables func (c *IcebergConnector) SetupQRepMetadataTables(_ context.Context, config *protos.QRepConfig) error { c.logger.Info("QRep metadata setup not needed for S3.") @@ -142,8 +115,9 @@ func (c *IcebergConnector) SetupQRepMetadataTables(_ context.Context, config *pr } // S3 doesn't check if partition is already synced, but file with same name is overwritten -func (c *IcebergConnector) IsQRepPartitionSynced(_ context.Context, +func (c *IcebergConnector) IsQRepPartitionSynced(ctx context.Context, config *protos.IsQRepPartitionSyncedInput, ) (bool, error) { - return false, nil + // TODO look at this + return c.PostgresMetadata.IsQRepPartitionSynced(ctx, config) } diff --git a/flow/connectors/s3/qrep.go b/flow/connectors/s3/qrep.go index 14c7b31ef2..943195b7b3 100644 --- a/flow/connectors/s3/qrep.go +++ b/flow/connectors/s3/qrep.go @@ -79,5 +79,6 @@ func (c *S3Connector) SetupQRepMetadataTables(_ context.Context, config *protos. func (c *S3Connector) IsQRepPartitionSynced(_ context.Context, config *protos.IsQRepPartitionSyncedInput, ) (bool, error) { + // TODO maybe we should actually check if the file exists instead of retrying blindly return false, nil } diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go index 764fd86759..d58d764ead 100644 --- a/flow/model/qvalue/avro_converter.go +++ b/flow/model/qvalue/avro_converter.go @@ -50,6 +50,14 @@ type AvroSchemaField struct { LogicalType string `json:"logicalType,omitempty"` } +// AvroSchemaFixed TODO this needs to be studied for Iceberg +type AvroSchemaFixed struct { + Type string `json:"type"` + Name string `json:"name"` + Size int `json:"size"` + LogicalType string `json:"logicalType,omitempty"` +} + func TruncateOrLogNumeric(num decimal.Decimal, precision int16, scale int16, targetDB protos.DBType) (decimal.Decimal, error) { if targetDB == protos.DBType_SNOWFLAKE || targetDB == protos.DBType_BIGQUERY { bidigi := datatypes.CountDigits(num.BigInt()) @@ -81,6 +89,16 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci case QValueKindInterval: return "string", nil case QValueKindUUID: + if targetDWH == protos.DBType_ICEBERG { + return "string", nil + // TODO use proper fixed uuids for iceberg as below + //return AvroSchemaFixed{ + // Type: "fixed", + // Size: 16, + // Name: "uuid_fixed_" + name, + // LogicalType: "uuid", + //}, nil + } return AvroSchemaLogical{ Type: "string", LogicalType: "uuid", @@ -120,12 +138,20 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci } return "string", nil case QValueKindTimestamp, QValueKindTimestampTZ: - if targetDWH == protos.DBType_CLICKHOUSE { + if targetDWH == protos.DBType_CLICKHOUSE || (targetDWH == protos.DBType_ICEBERG && kind == QValueKindTimestamp) { return AvroSchemaLogical{ Type: "long", LogicalType: "timestamp-micros", }, nil } + if targetDWH == protos.DBType_ICEBERG { + // This is specific to Iceberg, to enable timestamp with timezone + return map[string]interface{}{ + "type": "long", + "logicalType": "timestamp-micros", + "adjust-to-utc": true, + }, nil + } return "string", nil case QValueKindHStore, QValueKindJSON, QValueKindStruct: return "string", nil @@ -353,7 +379,17 @@ func QValueToAvro(value QValue, field *QField, targetDWH protos.DBType, logger l case QValueArrayDate: return c.processArrayDate(v.Val), nil case QValueUUID: - return c.processUUID(v.Val), nil + if c.TargetDWH == protos.DBType_ICEBERG { + // TODO make this a fixed type for iceberg uuids + //return c.processUUID(v.Val, "uuid_fixed_"+field.Name), nil + genUuid, err := uuid.FromBytes(v.Val[:]) + if err != nil { + return nil, fmt.Errorf("failed to convert UUID to string: %w", err) + } + return c.processNullableUnion("string", genUuid.String()) + + } + return c.processUUIDString(v.Val), nil case QValueGeography, QValueGeometry, QValuePoint: return c.processGeospatial(v.Value().(string)), nil default: @@ -548,14 +584,22 @@ func (c *QValueAvroConverter) processHStore(hstore string) (interface{}, error) return jsonString, nil } -func (c *QValueAvroConverter) processUUID(byteData [16]byte) interface{} { +func (c *QValueAvroConverter) processUUIDString(byteData [16]byte) interface{} { uuidString := uuid.UUID(byteData).String() if c.Nullable { - return goavro.Union("string", uuidString) + return goavro.Union("string.uuid", uuidString) } return uuidString } +func (c *QValueAvroConverter) processUUID(byteData [16]byte, uuidTypeName string) interface{} { + if c.Nullable { + // Slice is required by goavro + return goavro.Union(uuidTypeName, byteData[:]) + } + return byteData +} + func (c *QValueAvroConverter) processGeospatial(geoString string) interface{} { if c.Nullable { return goavro.Union("string", geoString) diff --git a/protos/flow-jvm.proto b/protos/flow-jvm.proto index ed30c4a207..6749f4134a 100644 --- a/protos/flow-jvm.proto +++ b/protos/flow-jvm.proto @@ -99,6 +99,7 @@ message AppendRecordsRequest { TableInfo table_info = 1; string schema = 2; repeated InsertRecord records = 3; + optional string idempotency_key = 4; } message AppendRecordsResponse { diff --git a/protos/peers.proto b/protos/peers.proto index 040cb29dbb..7f7cf03fa2 100644 --- a/protos/peers.proto +++ b/protos/peers.proto @@ -179,18 +179,26 @@ message IcebergS3IoConfig { optional string path_style_access = 5; } +message IcebergGCSIoConfig { + optional string project_id = 1; + optional string bucket = 2; + optional string credentials = 3; +} + message IcebergIOConfig { oneof config { IcebergS3IoConfig s3 = 1; + IcebergGCSIoConfig gcs = 2; } } message CommonIcebergCatalog { - string uri = 1; - string warehouse_location = 2; - optional int32 client_pool_size = 3; - optional bool cache_enabled = 4; - map additional_properties = 5; + string name = 1; + string uri = 2; + string warehouse_location = 3; + optional int32 client_pool_size = 4; + optional bool cache_enabled = 5; + map additional_properties = 6; } message HiveIcebergCatalog {} From 68af0b938a8c5964d71701e632dda703459f6bc3 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 11 Jun 2024 09:33:03 +0530 Subject: [PATCH 03/31] chore: nit fixes --- flow-jvm/README.md | 6 +- .../jvm/iceberg/catalog/CatalogLoader.java | 6 +- flow/activities/flowable.go | 1 - flow/connectors/core.go | 2 +- flow/connectors/iceberg/iceberg.go | 133 ++++++++++++------ flow/connectors/iceberg/qrep.go | 21 ++- flow/model/qvalue/avro_converter.go | 13 +- flow/peerdbenv/config.go | 5 + 8 files changed, 118 insertions(+), 69 deletions(-) diff --git a/flow-jvm/README.md b/flow-jvm/README.md index eda52dd049..6377e5877f 100644 --- a/flow-jvm/README.md +++ b/flow-jvm/README.md @@ -11,7 +11,7 @@ You can run your application in dev mode that enables live coding using: ./gradlew quarkusDev ``` -> **_NOTE:_** Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:8080/q/dev/. +> **_NOTE:_** Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:9801/q/dev/. ## Packaging and running the application @@ -33,12 +33,12 @@ The application, packaged as an _über-jar_, is now runnable using `java -jar bu ## Creating a native executable -You can create a native executable using: +You can create a native executable using: ```shell script ./gradlew build -Dquarkus.native.enabled=true ``` -Or, if you don't have GraalVM installed, you can run the native executable build in a container using: +Or, if you don't have GraalVM installed, you can run the native executable build in a container using: ```shell script ./gradlew build -Dquarkus.native.enabled=true -Dquarkus.native.container-build=true ``` diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java index b6da3596f5..d4b2d3c749 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java @@ -41,15 +41,15 @@ public Catalog loadCatalog(IcebergCatalog icebergCatalogConfig) { } }; - var catalogconfig = switch (icebergCatalogConfig.getConfigCase()) { + var catalogConfig = switch (icebergCatalogConfig.getConfigCase()) { case HIVE -> hiveConfigMapper.map(icebergCatalogConfig.getCommonConfig(), icebergCatalogConfig.getHive(), fileIoConfig); case JDBC -> jdbcCatalogMapper.map(icebergCatalogConfig.getCommonConfig(), icebergCatalogConfig.getJdbc(), fileIoConfig); default -> - throw new IllegalStateException("Unexpected value for catalog config: " + icebergCatalogConfig.getConfigCase()); + throw new IllegalArgumentException("Unexpected value for catalog config: " + icebergCatalogConfig.getConfigCase()); }; // TODO look at hadoop - return CatalogUtil.buildIcebergCatalog(icebergCatalogConfig.getCommonConfig().getName(), catalogconfig, null); + return CatalogUtil.buildIcebergCatalog(icebergCatalogConfig.getCommonConfig().getName(), catalogConfig, null); } } diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go index b866f6be67..b5c3c539f8 100644 --- a/flow/activities/flowable.go +++ b/flow/activities/flowable.go @@ -923,7 +923,6 @@ func (a *FlowableActivity) LoadPeer(ctx context.Context, peerName string) (*prot var config protos.IcebergConfig if err := proto.Unmarshal(peerOptions, &config); err != nil { return nil, fmt.Errorf("failed to unmarshal Iceberg config: %w", err) - } peer.Config = &protos.Peer_IcebergConfig{IcebergConfig: &config} default: diff --git a/flow/connectors/core.go b/flow/connectors/core.go index 5200d742d4..ea8028e4a2 100644 --- a/flow/connectors/core.go +++ b/flow/connectors/core.go @@ -3,7 +3,6 @@ package connectors import ( "context" "errors" - "github.com/PeerDB-io/peer-flow/connectors/iceberg" "log/slog" "github.com/jackc/pgx/v5/pgxpool" @@ -13,6 +12,7 @@ import ( connclickhouse "github.com/PeerDB-io/peer-flow/connectors/clickhouse" connelasticsearch "github.com/PeerDB-io/peer-flow/connectors/connelasticsearch" conneventhub "github.com/PeerDB-io/peer-flow/connectors/eventhub" + "github.com/PeerDB-io/peer-flow/connectors/iceberg" connkafka "github.com/PeerDB-io/peer-flow/connectors/kafka" connmysql "github.com/PeerDB-io/peer-flow/connectors/mysql" connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go index 0ba1714579..48d309fb9d 100644 --- a/flow/connectors/iceberg/iceberg.go +++ b/flow/connectors/iceberg/iceberg.go @@ -3,23 +3,24 @@ package iceberg import ( "context" "fmt" - "github.com/PeerDB-io/peer-flow/alerting" - "github.com/PeerDB-io/peer-flow/datatypes" - "github.com/PeerDB-io/peer-flow/model/qvalue" - "github.com/PeerDB-io/peer-flow/otel_metrics/peerdb_guages" - "github.com/jackc/pgx/v5/pgxpool" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "log/slog" "time" + "github.com/jackc/pgx/v5/pgxpool" "go.temporal.io/sdk/log" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "github.com/PeerDB-io/peer-flow/alerting" metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" + "github.com/PeerDB-io/peer-flow/datatypes" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/otel_metrics/peerdb_guages" + "github.com/PeerDB-io/peer-flow/peerdbenv" ) type IcebergConnector struct { @@ -30,63 +31,80 @@ type IcebergConnector struct { proxyClient protos.IcebergProxyServiceClient } -func (c *IcebergConnector) GetTableSchema(ctx context.Context, req *protos.GetTableSchemaBatchInput) (*protos.GetTableSchemaBatchOutput, error) { - //TODO implement me +func (c *IcebergConnector) GetTableSchema( + ctx context.Context, + req *protos.GetTableSchemaBatchInput, +) (*protos.GetTableSchemaBatchOutput, error) { + // TODO implement me panic("implement me") } -func (c *IcebergConnector) EnsurePullability(ctx context.Context, req *protos.EnsurePullabilityBatchInput) (*protos.EnsurePullabilityBatchOutput, error) { - //TODO implement me +func (c *IcebergConnector) EnsurePullability( + ctx context.Context, + req *protos.EnsurePullabilityBatchInput, +) (*protos.EnsurePullabilityBatchOutput, error) { + // TODO implement me panic("implement me") } func (c *IcebergConnector) ExportTxSnapshot(ctx context.Context) (*protos.ExportTxSnapshotOutput, any, error) { - //TODO implement me + // TODO implement me panic("implement me") } func (c *IcebergConnector) FinishExport(a any) error { - //TODO implement me + // TODO implement me panic("implement me") } func (c *IcebergConnector) SetupReplConn(ctx context.Context) error { - //TODO implement me + // TODO implement me panic("implement me") } func (c *IcebergConnector) ReplPing(ctx context.Context) error { - //TODO implement me + // TODO implement me panic("implement me") } func (c *IcebergConnector) UpdateReplStateLastOffset(lastOffset int64) { - //TODO implement me + // TODO implement me panic("implement me") } func (c *IcebergConnector) PullFlowCleanup(ctx context.Context, jobName string) error { - //TODO implement me + // TODO implement me panic("implement me") } -func (c *IcebergConnector) HandleSlotInfo(ctx context.Context, alerter *alerting.Alerter, catalogPool *pgxpool.Pool, slotName string, peerName string, slotMetricGuages peerdb_guages.SlotMetricGuages) error { - //TODO implement me +func (c *IcebergConnector) HandleSlotInfo( + ctx context.Context, + alerter *alerting.Alerter, + catalogPool *pgxpool.Pool, + slotName string, + peerName string, + slotMetricGuages peerdb_guages.SlotMetricGuages, +) error { + // TODO implement me panic("implement me") } func (c *IcebergConnector) GetSlotInfo(ctx context.Context, slotName string) ([]*protos.SlotInfo, error) { - //TODO implement me + // TODO implement me panic("implement me") } func (c *IcebergConnector) AddTablesToPublication(ctx context.Context, req *protos.AddTablesToPublicationInput) error { - //TODO implement me + // TODO implement me panic("implement me") } -func (c *IcebergConnector) PullRecords(ctx context.Context, catalogPool *pgxpool.Pool, req *model.PullRecordsRequest[model.RecordItems]) error { - //TODO implement me +func (c *IcebergConnector) PullRecords( + ctx context.Context, + catalogPool *pgxpool.Pool, + req *model.PullRecordsRequest[model.RecordItems], +) error { + // TODO implement me panic("implement me") } @@ -95,10 +113,7 @@ func (c *IcebergConnector) StartSetupNormalizedTables(ctx context.Context) (any, return nil, nil } -func (c *IcebergConnector) CleanupSetupNormalizedTables(ctx context.Context, tx any) { - return - -} +func (c *IcebergConnector) CleanupSetupNormalizedTables(ctx context.Context, tx any) {} func (c *IcebergConnector) FinishSetupNormalizedTables(ctx context.Context, tx any) error { return nil @@ -114,7 +129,14 @@ func (c *IcebergConnector) SyncFlowCleanup(ctx context.Context, jobName string) return nil } -func (c *IcebergConnector) SetupNormalizedTable(ctx context.Context, tx any, tableIdentifier string, tableSchema *protos.TableSchema, softDeleteColName string, syncedAtColName string) (bool, error) { +func (c *IcebergConnector) SetupNormalizedTable( + ctx context.Context, + tx any, + tableIdentifier string, + tableSchema *protos.TableSchema, + softDeleteColName string, + syncedAtColName string, +) (bool, error) { // TODO add soft delete column in the schema qFields := make([]qvalue.QField, len(tableSchema.Columns)) for i, fieldDescription := range tableSchema.Columns { @@ -158,24 +180,23 @@ func (c *IcebergConnector) SetupNormalizedTable(ctx context.Context, tx any, tab } c.logger.Debug("Created iceberg table", slog.String("table", tableResponse.TableName)) // TODO need to re-enable this and see why it is failing - //if tableResponse.TableName != tableIdentifier { + // if tableResponse.TableName != tableIdentifier { // return false, fmt.Errorf("created table name mismatch: %s != %s", tableResponse.TableName, tableIdentifier) //} return true, nil } func addPeerMetaColumns(qFields []qvalue.QField, softDeleteColName string, syncedAtColName string) []qvalue.QField { - qFields = append(qFields, qvalue.QField{ - Name: softDeleteColName, - Type: qvalue.QValueKindBoolean, - Nullable: true, - }) - - qFields = append(qFields, qvalue.QField{ - Name: syncedAtColName, - Type: qvalue.QValueKindTimestampTZ, - Nullable: true, - }) + qFields = append(qFields, + qvalue.QField{ + Name: softDeleteColName, + Type: qvalue.QValueKindBoolean, + Nullable: true, + }, qvalue.QField{ + Name: syncedAtColName, + Type: qvalue.QValueKindTimestampTZ, + Nullable: true, + }) return qFields } @@ -184,7 +205,10 @@ func NewIcebergConnector( config *protos.IcebergConfig, ) (*IcebergConnector, error) { logger := logger.LoggerFromCtx(ctx) - conn, err := grpc.NewClient("localhost:9801", grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient( + peerdbenv.PeerDBFlowJvmAddress(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) if err != nil { return nil, fmt.Errorf("failed to connect to Iceberg proxy: %w", err) } @@ -206,7 +230,6 @@ func NewIcebergConnector( } func (c *IcebergConnector) CreateRawTable(_ context.Context, req *protos.CreateRawTableInput) (*protos.CreateRawTableOutput, error) { - // TODO c.logger.Info("CreateRawTable for Iceberg is a no-op") return nil, nil } @@ -227,7 +250,29 @@ func (c *IcebergConnector) ValidateCheck(ctx context.Context) error { IcebergCatalog: c.config.CatalogConfig, PrimaryKey: nil, }, - Schema: "{\n \"type\": \"record\",\n \"name\": \"TestObject\",\n \"namespace\": \"ca.dataedu\",\n \"fields\": [\n {\n \"name\": \"hello\",\n \"type\": [\n \"null\",\n \"int\"\n ],\n \"default\": null\n },\n {\n \"name\": \"some\",\n \"type\": [\n \"null\",\n \"string\"\n ],\n \"default\": null\n }\n ]\n}", + Schema: `{ + "type": "record", + "name": "TestObject", + "namespace": "", + "fields": [ + { + "name": "hello", + "type": [ + "null", + "int" + ], + "default": null + }, + { + "name": "some", + "type": [ + "null", + "string" + ], + "default": null + } + ] +}`, }) if err != nil { return err @@ -256,7 +301,7 @@ func (c *IcebergConnector) ValidateCheck(ctx context.Context) error { } func (c *IcebergConnector) ConnectionActive(ctx context.Context) error { - // TODO + // TODO implement this for iceberg return nil } diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go index 60f0f766ec..7df2646744 100644 --- a/flow/connectors/iceberg/qrep.go +++ b/flow/connectors/iceberg/qrep.go @@ -3,12 +3,13 @@ package iceberg import ( "context" "fmt" - "github.com/PeerDB-io/peer-flow/logger" - "github.com/linkedin/goavro/v2" "log/slog" "time" + "github.com/linkedin/goavro/v2" + "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" ) @@ -41,15 +42,13 @@ func (c *IcebergConnector) SyncQRepRecords( } binaryRecords := make([]*protos.InsertRecord, 0) for record := range stream.Records { - // Add soft delete record = append(record, qvalue.QValueBoolean{ Val: false, - }) - // add synced at colname - record = append(record, qvalue.QValueTimestampTZ{ - Val: time.Now(), - }) + }, // add synced at colname + qvalue.QValueTimestampTZ{ + Val: time.Now(), + }) converted, err := avroConverter.Convert(record) if err != nil { @@ -63,7 +62,6 @@ func (c *IcebergConnector) SyncQRepRecords( binaryRecords = append(binaryRecords, &protos.InsertRecord{ Record: native, }) - } requestIdempotencyKey := fmt.Sprintf("_peerdb_qrep-%s-%s", config.FlowJobName, partition.PartitionId) @@ -71,10 +69,10 @@ func (c *IcebergConnector) SyncQRepRecords( appendRecordsResponse, err := c.proxyClient.AppendRecords(ctx, &protos.AppendRecordsRequest{ TableInfo: &protos.TableInfo{ - //Namespace: nil, + // Namespace: nil, TableName: dstTableName, IcebergCatalog: c.config.CatalogConfig, - //PrimaryKey: nil, + // PrimaryKey: nil, }, Schema: avroSchema.Schema, @@ -82,7 +80,6 @@ func (c *IcebergConnector) SyncQRepRecords( IdempotencyKey: &requestIdempotencyKey, }, ) - if err != nil { return 0, err } diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go index d58d764ead..7c1d3c469e 100644 --- a/flow/model/qvalue/avro_converter.go +++ b/flow/model/qvalue/avro_converter.go @@ -54,8 +54,8 @@ type AvroSchemaField struct { type AvroSchemaFixed struct { Type string `json:"type"` Name string `json:"name"` - Size int `json:"size"` LogicalType string `json:"logicalType,omitempty"` + Size int `json:"size"` } func TruncateOrLogNumeric(num decimal.Decimal, precision int16, scale int16, targetDB protos.DBType) (decimal.Decimal, error) { @@ -92,12 +92,12 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci if targetDWH == protos.DBType_ICEBERG { return "string", nil // TODO use proper fixed uuids for iceberg as below - //return AvroSchemaFixed{ + // return AvroSchemaFixed{ // Type: "fixed", // Size: 16, // Name: "uuid_fixed_" + name, // LogicalType: "uuid", - //}, nil + // }, nil } return AvroSchemaLogical{ Type: "string", @@ -381,13 +381,12 @@ func QValueToAvro(value QValue, field *QField, targetDWH protos.DBType, logger l case QValueUUID: if c.TargetDWH == protos.DBType_ICEBERG { // TODO make this a fixed type for iceberg uuids - //return c.processUUID(v.Val, "uuid_fixed_"+field.Name), nil + // return c.processUUID(v.Val, "uuid_fixed_"+field.Name), nil genUuid, err := uuid.FromBytes(v.Val[:]) if err != nil { return nil, fmt.Errorf("failed to convert UUID to string: %w", err) } return c.processNullableUnion("string", genUuid.String()) - } return c.processUUIDString(v.Val), nil case QValueGeography, QValueGeometry, QValuePoint: @@ -592,6 +591,10 @@ func (c *QValueAvroConverter) processUUIDString(byteData [16]byte) interface{} { return uuidString } +// processUUID converts a UUID byte array to a string or a byte array based on the nullable flag +// TODO it needs to be used once we have fixed types for Iceberg +// +//nolint:unused func (c *QValueAvroConverter) processUUID(byteData [16]byte, uuidTypeName string) interface{} { if c.Nullable { // Slice is required by goavro diff --git a/flow/peerdbenv/config.go b/flow/peerdbenv/config.go index bcf1ac050e..c2d22ea919 100644 --- a/flow/peerdbenv/config.go +++ b/flow/peerdbenv/config.go @@ -145,3 +145,8 @@ func PeerDBAlertingEmailSenderReplyToAddresses() string { func PeerDBClickhouseAWSS3BucketName() string { return GetEnvString("PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME", "") } + +// PeerDBFlowJvmAddress is the URL of the gRPC server for the JVM-based proxy, Eg: "localhost:9801" +func PeerDBFlowJvmAddress() string { + return GetEnvString("PEERDB_FLOW_JVM_ADDRESS", "") +} From 4c88206c54bf80349e1268516142f0037a3866f8 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 11 Jun 2024 09:50:05 +0530 Subject: [PATCH 04/31] chore: some MORE nit fixes --- ...erter.java => AvroIcebergRecordConverter.java} | 6 +++--- .../flow/jvm/iceberg/service/IcebergService.java | 13 ++++++++----- flow/connectors/iceberg/qrep.go | 15 ++++++++------- flow/logger/handler.go | 3 --- flow/model/qvalue/avro_converter.go | 2 +- flow/peerdbenv/config.go | 5 ----- nexus/analyzer/src/lib.rs | 4 ++++ nexus/catalog/src/lib.rs | 6 ++++++ protos/peers.proto | 3 ++- 9 files changed, 32 insertions(+), 25 deletions(-) rename flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/{AvroIcebergConverter.java => AvroIcebergRecordConverter.java} (82%) diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergRecordConverter.java similarity index 82% rename from flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java rename to flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergRecordConverter.java index e559924f33..9c9407c442 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergConverter.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/avro/AvroIcebergRecordConverter.java @@ -8,17 +8,17 @@ import java.io.IOException; -public class AvroIcebergConverter { +public class AvroIcebergRecordConverter { private final org.apache.iceberg.Schema icebergSchema; private final Schema icebergAvroSchema; private final DataReader dataReader; - public AvroIcebergConverter(String avroSchemaString, org.apache.iceberg.Schema icebergSchema, String tableName) { + public AvroIcebergRecordConverter(String avroSchemaString, org.apache.iceberg.Schema icebergSchema, String tableName) { this(new Schema.Parser().parse(avroSchemaString), icebergSchema, tableName); } - public AvroIcebergConverter(Schema sourceAvroSchema, org.apache.iceberg.Schema icebergSchema, String tableName) { + public AvroIcebergRecordConverter(Schema sourceAvroSchema, org.apache.iceberg.Schema icebergSchema, String tableName) { this.icebergSchema = icebergSchema; this.icebergAvroSchema = AvroSchemaUtil.convert(icebergSchema, tableName); this.dataReader = DataReader.create(icebergSchema, icebergAvroSchema); diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index 7ab4911631..56368428af 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -4,7 +4,7 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Streams; import io.peerdb.flow.jvm.grpc.*; -import io.peerdb.flow.jvm.iceberg.avro.AvroIcebergConverter; +import io.peerdb.flow.jvm.iceberg.avro.AvroIcebergRecordConverter; import io.peerdb.flow.jvm.iceberg.catalog.CatalogLoader; import io.peerdb.flow.jvm.iceberg.lock.LockManager; import io.peerdb.flow.jvm.iceberg.writer.RecordWriterFactory; @@ -42,7 +42,7 @@ public class IcebergService { @Inject RecordWriterFactory recordWriterFactory; - private static void writeRecordStream(Stream recordStream, AvroIcebergConverter converter, TaskWriter writer) { + private static void writeRecordStream(Stream recordStream, AvroIcebergRecordConverter converter, TaskWriter writer) { recordStream.parallel().map(insertRecord -> { try { return converter.toIcebergRecord(insertRecord.getRecord().toByteArray()); @@ -108,8 +108,11 @@ private boolean appendRecords(TableInfo tableInfo, String avroSchema, List recordStream) { WriteResult writeResult; try (var writer = recordWriterFactory.createRecordWriter(table)) { - var converter = new AvroIcebergConverter(avroSchema, table.schema(), table.name()); + var converter = new AvroIcebergRecordConverter(avroSchema, table.schema(), table.name()); Log.infof("Will now write records to append to table %s", table.name()); var stopwatch = Stopwatch.createStarted(); writeRecordStream(recordStream, converter, writer); @@ -222,7 +225,7 @@ public boolean insertChanges(TableInfo tableInfo, String avroSchema, List { switch (recordChange.getChangeCase()) { case INSERT: diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go index 7df2646744..4b8ca14bc2 100644 --- a/flow/connectors/iceberg/qrep.go +++ b/flow/connectors/iceberg/qrep.go @@ -42,10 +42,11 @@ func (c *IcebergConnector) SyncQRepRecords( } binaryRecords := make([]*protos.InsertRecord, 0) for record := range stream.Records { - // Add soft delete - record = append(record, qvalue.QValueBoolean{ - Val: false, - }, // add synced at colname + record = append(record, + // Add soft delete + qvalue.QValueBoolean{ + Val: false, + }, // add synced at colname qvalue.QValueTimestampTZ{ Val: time.Now(), }) @@ -105,13 +106,13 @@ func getAvroSchema( return avroSchema, nil } -// S3 just sets up destination, not metadata tables +// Iceberg just sets up destination, not metadata tables func (c *IcebergConnector) SetupQRepMetadataTables(_ context.Context, config *protos.QRepConfig) error { - c.logger.Info("QRep metadata setup not needed for S3.") + c.logger.Info("QRep metadata setup not needed for Iceberg.") return nil } -// S3 doesn't check if partition is already synced, but file with same name is overwritten +// Iceberg doesn't check if partition is already synced, but file with same name is overwritten func (c *IcebergConnector) IsQRepPartitionSynced(ctx context.Context, config *protos.IsQRepPartitionSyncedInput, ) (bool, error) { diff --git a/flow/logger/handler.go b/flow/logger/handler.go index 8103fe7d4d..3351787228 100644 --- a/flow/logger/handler.go +++ b/flow/logger/handler.go @@ -23,9 +23,6 @@ func NewHandler(handler slog.Handler) slog.Handler { } func (h Handler) Enabled(ctx context.Context, level slog.Level) bool { - if level == slog.LevelDebug { - return true - } return h.handler.Enabled(ctx, level) } diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go index 7c1d3c469e..d7b1192eb1 100644 --- a/flow/model/qvalue/avro_converter.go +++ b/flow/model/qvalue/avro_converter.go @@ -586,7 +586,7 @@ func (c *QValueAvroConverter) processHStore(hstore string) (interface{}, error) func (c *QValueAvroConverter) processUUIDString(byteData [16]byte) interface{} { uuidString := uuid.UUID(byteData).String() if c.Nullable { - return goavro.Union("string.uuid", uuidString) + return goavro.Union("string", uuidString) } return uuidString } diff --git a/flow/peerdbenv/config.go b/flow/peerdbenv/config.go index 4f75c0ea19..bba3ae2d04 100644 --- a/flow/peerdbenv/config.go +++ b/flow/peerdbenv/config.go @@ -93,11 +93,6 @@ func PeerDBAlertingEmailSenderReplyToAddresses() string { return GetEnvString("PEERDB_ALERTING_EMAIL_SENDER_REPLY_TO_ADDRESSES", "") } -// PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME -func PeerDBClickhouseAWSS3BucketName() string { - return GetEnvString("PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME", "") -} - // PeerDBFlowJvmAddress is the URL of the gRPC server for the JVM-based proxy, Eg: "localhost:9801" func PeerDBFlowJvmAddress() string { return GetEnvString("PEERDB_FLOW_JVM_ADDRESS", "") diff --git a/nexus/analyzer/src/lib.rs b/nexus/analyzer/src/lib.rs index 4548d2eec9..a23cc6caf2 100644 --- a/nexus/analyzer/src/lib.rs +++ b/nexus/analyzer/src/lib.rs @@ -966,5 +966,9 @@ fn parse_db_options(db_type: DbType, with_options: &[SqlOption]) -> anyhow::Resu .and_then(|s| s.parse::().ok()) .unwrap_or_default(), }), + // TODO complete this for iceberg once finalized + DbType::Iceberg => Config::IcebergConfig(pt::peerdb_peers::IcebergConfig { + catalog_config: None, + }), })) } diff --git a/nexus/catalog/src/lib.rs b/nexus/catalog/src/lib.rs index a817ba3bd8..56ee2f63dc 100644 --- a/nexus/catalog/src/lib.rs +++ b/nexus/catalog/src/lib.rs @@ -105,6 +105,7 @@ impl Catalog { elasticsearch_config.encode_to_vec() } Config::MysqlConfig(mysql_config) => mysql_config.encode_to_vec(), + Config::IcebergConfig(iceberg_config) => {iceberg_config.encode_to_vec()} } }; @@ -334,6 +335,11 @@ impl Catalog { pt::peerdb_peers::MySqlConfig::decode(options).with_context(err)?; Config::MysqlConfig(mysql_config) } + DbType::Iceberg => { + let iceberg_config = + pt::peerdb_peers::IcebergConfig::decode(options).with_context(err)?; + Config::IcebergConfig(iceberg_config) + } }) } else { None diff --git a/protos/peers.proto b/protos/peers.proto index 7f7cf03fa2..efc5d4309e 100644 --- a/protos/peers.proto +++ b/protos/peers.proto @@ -198,7 +198,8 @@ message CommonIcebergCatalog { string warehouse_location = 3; optional int32 client_pool_size = 4; optional bool cache_enabled = 5; - map additional_properties = 6; + // This helps in testing, where we can pass additional properties to the catalog +// map additional_properties = 6; } message HiveIcebergCatalog {} From 004c9891d565cc534260f4b48318706472817678 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:05:15 +0530 Subject: [PATCH 05/31] chore: update quarkus cli version --- .sdkmanrc | 2 +- flow-jvm/.sdkmanrc | 1 + flow-jvm/README.md | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 120000 flow-jvm/.sdkmanrc diff --git a/.sdkmanrc b/.sdkmanrc index 8d3433b204..860bdeb31b 100644 --- a/.sdkmanrc +++ b/.sdkmanrc @@ -1,4 +1,4 @@ # Enable auto-env through the sdkman_auto_env config # Add key=value pairs of SDKs to use below java=21.0.3-tem -quarkus=3.10.2 +quarkus=3.11.1 diff --git a/flow-jvm/.sdkmanrc b/flow-jvm/.sdkmanrc new file mode 120000 index 0000000000..f0da53ae97 --- /dev/null +++ b/flow-jvm/.sdkmanrc @@ -0,0 +1 @@ +../.sdkmanrc \ No newline at end of file diff --git a/flow-jvm/README.md b/flow-jvm/README.md index 6377e5877f..f69c873b94 100644 --- a/flow-jvm/README.md +++ b/flow-jvm/README.md @@ -1,5 +1,11 @@ # flow-jvm + +## Dependencies +Install sdkman and run `sdk env install` to setup the environment. + + + This project uses Quarkus, the Supersonic Subatomic Java Framework. If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ . From 225e30b673a9230e848d5254536916476e2a8034 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:31:19 +0530 Subject: [PATCH 06/31] chore: fix pr comments --- .sdkmanrc | 1 + flow-jvm/.gitignore | 2 + flow-jvm/README.md | 2 +- flow-jvm/gradle/wrapper/gradle-wrapper.jar | Bin 58910 -> 0 bytes .../gradle/wrapper/gradle-wrapper.properties | 2 + flow-jvm/gradlew | 282 +++++++++++------- flow-jvm/gradlew.bat | 54 ++-- .../java/io/peerdb/IcebergServiceTest.java | 22 -- flow/connectors/iceberg/iceberg.go | 3 +- 9 files changed, 201 insertions(+), 167 deletions(-) delete mode 100644 flow-jvm/gradle/wrapper/gradle-wrapper.jar delete mode 100644 flow-jvm/src/test/java/io/peerdb/IcebergServiceTest.java diff --git a/.sdkmanrc b/.sdkmanrc index 860bdeb31b..00e6432dd9 100644 --- a/.sdkmanrc +++ b/.sdkmanrc @@ -2,3 +2,4 @@ # Add key=value pairs of SDKs to use below java=21.0.3-tem quarkus=3.11.1 +gradle=8.6 diff --git a/flow-jvm/.gitignore b/flow-jvm/.gitignore index 216783d79c..1c48a80737 100644 --- a/flow-jvm/.gitignore +++ b/flow-jvm/.gitignore @@ -37,3 +37,5 @@ nb-configuration.xml # Plugin directory /.quarkus/cli/plugins/ + +*.jar diff --git a/flow-jvm/README.md b/flow-jvm/README.md index f69c873b94..73983d501a 100644 --- a/flow-jvm/README.md +++ b/flow-jvm/README.md @@ -2,7 +2,7 @@ ## Dependencies -Install sdkman and run `sdk env install` to setup the environment. +Install sdkman and run `sdk env install` to setup the environment and run `gradle wrapper` to install gradle wrapper. diff --git a/flow-jvm/gradle/wrapper/gradle-wrapper.jar b/flow-jvm/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 62d4c053550b91381bbd28b1afc82d634bf73a8a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 58910 zcma&ObC74zk}X`WF59+k+qTVL*+!RbS9RI8Z5v&-ZFK4Nn|tqzcjwK__x+Iv5xL`> zj94dg?X`0sMHx^qXds{;KY)OMg#H>35XgTVfq6#vc9ww|9) z@UMfwUqk)B9p!}NrNqTlRO#i!ALOPcWo78-=iy}NsAr~T8T0X0%G{DhX~u-yEwc29WQ4D zuv2j{a&j?qB4wgCu`zOXj!~YpTNFg)TWoV>DhYlR^Gp^rkOEluvxkGLB?!{fD!T@( z%3cy>OkhbIKz*R%uoKqrg1%A?)uTZD&~ssOCUBlvZhx7XHQ4b7@`&sPdT475?*zWy z>xq*iK=5G&N6!HiZaD{NSNhWL;+>Quw_#ZqZbyglna!Fqn3N!$L`=;TFPrhodD-Q` z1l*=DP2gKJP@)cwI@-M}?M$$$%u~=vkeC%>cwR$~?y6cXx-M{=wdT4|3X(@)a|KkZ z`w$6CNS@5gWS7s7P86L<=vg$Mxv$?)vMj3`o*7W4U~*Nden}wz=y+QtuMmZ{(Ir1D zGp)ZsNiy{mS}Au5;(fYf93rs^xvi(H;|H8ECYdC`CiC&G`zw?@)#DjMc7j~daL_A$ z7e3nF2$TKlTi=mOftyFBt8*Xju-OY@2k@f3YBM)-v8+5_o}M?7pxlNn)C0Mcd@87?+AA4{Ti2ptnYYKGp`^FhcJLlT%RwP4k$ad!ho}-^vW;s{6hnjD0*c39k zrm@PkI8_p}mnT&5I@=O1^m?g}PN^8O8rB`;t`6H+?Su0IR?;8txBqwK1Au8O3BZAX zNdJB{bpQWR@J|e=Z>XSXV1DB{uhr3pGf_tb)(cAkp)fS7*Qv))&Vkbb+cvG!j}ukd zxt*C8&RN}5ck{jkw0=Q7ldUp0FQ&Pb_$M7a@^nf`8F%$ftu^jEz36d#^M8Ia{VaTy z5(h$I)*l3i!VpPMW+XGgzL~fcN?{~1QWu9!Gu0jOWWE zNW%&&by0DbXL&^)r-A*7R@;T$P}@3eOj#gqJ!uvTqBL5bupU91UK#d|IdxBUZAeh1 z>rAI#*Y4jv>uhOh7`S@mnsl0g@1C;k$Z%!d*n8#_$)l}-1&z2kr@M+xWoKR z!KySy-7h&Bf}02%JeXmQGjO3ntu={K$jy$rFwfSV8!zqAL_*&e2|CJ06`4&0+ceI026REfNT>JzAdwmIlKLEr2? zaZ#d*XFUN*gpzOxq)cysr&#6zNdDDPH% zd8_>3B}uA7;bP4fKVdd~Og@}dW#74ceETOE- zlZgQqQfEc?-5ly(Z5`L_CCM!&Uxk5#wgo=OLs-kFHFG*cTZ)$VE?c_gQUW&*!2@W2 z7Lq&_Kf88OCo?BHCtwe*&fu&8PQ(R5&lnYo8%+U73U)Ec2&|A)Y~m7(^bh299REPe zn#gyaJ4%o4>diN3z%P5&_aFUmlKytY$t21WGwx;3?UC}vlxi-vdEQgsKQ;=#sJ#ll zZeytjOad$kyON4XxC}frS|Ybh`Yq!<(IrlOXP3*q86ImyV*mJyBn$m~?#xp;EplcM z+6sez%+K}Xj3$YN6{}VL;BZ7Fi|iJj-ywlR+AP8lq~mnt5p_%VmN{Sq$L^z!otu_u znVCl@FgcVXo510e@5(wnko%Pv+^r^)GRh;>#Z(|#cLnu_Y$#_xG&nvuT+~gzJsoSi zBvX`|IS~xaold!`P!h(v|=>!5gk)Q+!0R1Ge7!WpRP{*Ajz$oGG$_?Ajvz6F0X?809o`L8prsJ*+LjlGfSziO;+ zv>fyRBVx#oC0jGK8$%$>Z;0+dfn8x;kHFQ?Rpi7(Rc{Uq{63Kgs{IwLV>pDK7yX-2 zls;?`h!I9YQVVbAj7Ok1%Y+F?CJa-Jl>1x#UVL(lpzBBH4(6v0^4 z3Tf`INjml5`F_kZc5M#^J|f%7Hgxg3#o}Zwx%4l9yYG!WaYUA>+dqpRE3nw#YXIX%= ziH3iYO~jr0nP5xp*VIa#-aa;H&%>{mfAPPlh5Fc!N7^{!z$;p-p38aW{gGx z)dFS62;V;%%fKp&i@+5x=Cn7Q>H`NofJGXmNeh{sOL+Nk>bQJJBw3K*H_$}%*xJM=Kh;s#$@RBR z|75|g85da@#qT=pD777m$wI!Q8SC4Yw3(PVU53bzzGq$IdGQoFb-c_(iA_~qD|eAy z@J+2!tc{|!8fF;%6rY9`Q!Kr>MFwEH%TY0y>Q(D}xGVJM{J{aGN0drG&|1xO!Ttdw z-1^gQ&y~KS5SeslMmoA$Wv$ly={f}f9<{Gm!8ycp*D9m*5Ef{ymIq!MU01*)#J1_! zM_i4{LYButqlQ>Q#o{~W!E_#(S=hR}kIrea_67Z5{W>8PD>g$f;dTvlD=X@T$8D0;BWkle@{VTd&D5^)U>(>g(jFt4lRV6A2(Te->ooI{nk-bZ(gwgh zaH4GT^wXPBq^Gcu%xW#S#p_&x)pNla5%S5;*OG_T^PhIIw1gXP&u5c;{^S(AC*+$> z)GuVq(FT@zq9;i{*9lEsNJZ)??BbSc5vF+Kdh-kL@`(`l5tB4P!9Okin2!-T?}(w% zEpbEU67|lU#@>DppToestmu8Ce=gz=e#V+o)v)#e=N`{$MI5P0O)_fHt1@aIC_QCv=FO`Qf=Ga%^_NhqGI)xtN*^1n{ z&vgl|TrKZ3Vam@wE0p{c3xCCAl+RqFEse@r*a<3}wmJl-hoJoN<|O2zcvMRl<#BtZ z#}-bPCv&OTw`GMp&n4tutf|er`@#d~7X+);##YFSJ)BitGALu}-N*DJdCzs(cQ?I- z6u(WAKH^NUCcOtpt5QTsQRJ$}jN28ZsYx+4CrJUQ%egH zo#tMoywhR*oeIkS%}%WUAIbM`D)R6Ya&@sZvvUEM7`fR0Ga03*=qaEGq4G7-+30Ck zRkje{6A{`ebq?2BTFFYnMM$xcQbz0nEGe!s%}O)m={`075R0N9KTZ>vbv2^eml>@}722%!r#6Wto}?vNst? zs`IasBtcROZG9+%rYaZe^=5y3chDzBf>;|5sP0!sP(t^= z^~go8msT@|rp8LJ8km?4l?Hb%o10h7(ixqV65~5Y>n_zG3AMqM3UxUNj6K-FUgMT7 z*Dy2Y8Ws+%`Z*~m9P zCWQ8L^kA2$rf-S@qHow$J86t)hoU#XZ2YK~9GXVR|*`f6`0&8j|ss_Ai-x=_;Df^*&=bW$1nc{Gplm zF}VF`w)`5A;W@KM`@<9Bw_7~?_@b{Z`n_A6c1AG#h#>Z$K>gX6reEZ*bZRjCup|0# zQ{XAb`n^}2cIwLTN%5Ix`PB*H^(|5S{j?BwItu+MS`1)VW=TnUtt6{3J!WR`4b`LW z?AD#ZmoyYpL=903q3LSM=&5eNP^dwTDRD~iP=}FXgZ@2WqfdyPYl$9do?wX{RU*$S zgQ{OqXK-Yuf4+}x6P#A*la&^G2c2TC;aNNZEYuB(f25|5eYi|rd$;i0qk7^3Ri8of ziP~PVT_|4$n!~F-B1_Et<0OJZ*e+MN;5FFH`iec(lHR+O%O%_RQhvbk-NBQ+$)w{D+dlA0jxI;z|P zEKW`!X)${xzi}Ww5G&@g0akBb_F`ziv$u^hs0W&FXuz=Ap>SUMw9=M?X$`lgPRq11 zqq+n44qL;pgGO+*DEc+Euv*j(#%;>p)yqdl`dT+Og zZH?FXXt`<0XL2@PWYp|7DWzFqxLK)yDXae&3P*#+f+E{I&h=$UPj;ey9b`H?qe*Oj zV|-qgI~v%&oh7rzICXfZmg$8$B|zkjliQ=e4jFgYCLR%yi!9gc7>N z&5G#KG&Hr+UEfB;M(M>$Eh}P$)<_IqC_WKOhO4(cY@Gn4XF(#aENkp&D{sMQgrhDT zXClOHrr9|POHqlmm+*L6CK=OENXbZ+kb}t>oRHE2xVW<;VKR@ykYq04LM9L-b;eo& zl!QQo!Sw{_$-qosixZJWhciN>Gbe8|vEVV2l)`#5vKyrXc6E`zmH(76nGRdL)pqLb@j<&&b!qJRLf>d`rdz}^ZSm7E;+XUJ ziy;xY&>LM?MA^v0Fu8{7hvh_ynOls6CI;kQkS2g^OZr70A}PU;i^~b_hUYN1*j-DD zn$lHQG9(lh&sDii)ip*{;Sb_-Anluh`=l~qhqbI+;=ZzpFrRp&T+UICO!OoqX@Xr_ z32iJ`xSpx=lDDB_IG}k+GTYG@K8{rhTS)aoN8D~Xfe?ul&;jv^E;w$nhu-ICs&Q)% zZ=~kPNZP0-A$pB8)!`TEqE`tY3Mx^`%O`?EDiWsZpoP`e-iQ#E>fIyUx8XN0L z@S-NQwc;0HjSZKWDL}Au_Zkbh!juuB&mGL0=nO5)tUd_4scpPy&O7SNS^aRxUy0^< zX}j*jPrLP4Pa0|PL+nrbd4G;YCxCK-=G7TG?dby~``AIHwxqFu^OJhyIUJkO0O<>_ zcpvg5Fk$Wpj}YE3;GxRK67P_Z@1V#+pu>pRj0!mFf(m_WR3w3*oQy$s39~U7Cb}p(N&8SEwt+)@%o-kW9Ck=^?tvC2$b9% ze9(Jn+H`;uAJE|;$Flha?!*lJ0@lKfZM>B|c)3lIAHb;5OEOT(2453m!LgH2AX=jK zQ93An1-#l@I@mwB#pLc;M7=u6V5IgLl>E%gvE|}Hvd4-bE1>gs(P^C}gTv*&t>W#+ zASLRX$y^DD3Jrht zwyt`yuA1j(TcP*0p*Xkv>gh+YTLrcN_HuaRMso~0AJg`^nL#52dGBzY+_7i)Ud#X) zVwg;6$WV20U2uyKt8<)jN#^1>PLg`I`@Mmut*Zy!c!zshSA!e^tWVoKJD%jN&ml#{ z@}B$j=U5J_#rc%T7(DGKF+WwIblEZ;Vq;CsG~OKxhWYGJx#g7fxb-_ya*D0=_Ys#f zhXktl=Vnw#Z_neW>Xe#EXT(4sT^3p6srKby4Ma5LLfh6XrHGFGgM;5Z}jv-T!f~=jT&n>Rk z4U0RT-#2fsYCQhwtW&wNp6T(im4dq>363H^ivz#>Sj;TEKY<)dOQU=g=XsLZhnR>e zd}@p1B;hMsL~QH2Wq>9Zb; zK`0`09fzuYg9MLJe~cdMS6oxoAD{kW3sFAqDxvFM#{GpP^NU@9$d5;w^WgLYknCTN z0)N425mjsJTI@#2kG-kB!({*+S(WZ-{SckG5^OiyP%(6DpRsx60$H8M$V65a_>oME z^T~>oG7r!ew>Y)&^MOBrgc-3PezgTZ2xIhXv%ExMFgSf5dQbD=Kj*!J4k^Xx!Z>AW ziZfvqJvtm|EXYsD%A|;>m1Md}j5f2>kt*gngL=enh<>#5iud0dS1P%u2o+>VQ{U%(nQ_WTySY(s#~~> zrTsvp{lTSup_7*Xq@qgjY@1#bisPCRMMHnOL48qi*jQ0xg~TSW%KMG9zN1(tjXix()2$N}}K$AJ@GUth+AyIhH6Aeh7qDgt#t*`iF5#A&g4+ zWr0$h9Zx6&Uo2!Ztcok($F>4NA<`dS&Js%L+67FT@WmI)z#fF~S75TUut%V($oUHw z$IJsL0X$KfGPZYjB9jaj-LaoDD$OMY4QxuQ&vOGo?-*9@O!Nj>QBSA6n$Lx|^ zky)4+sy{#6)FRqRt6nM9j2Lzba!U;aL%ZcG&ki1=3gFx6(&A3J-oo|S2_`*w9zT)W z4MBOVCp}?4nY)1))SOX#6Zu0fQQ7V{RJq{H)S#;sElY)S)lXTVyUXTepu4N)n85Xo zIpWPT&rgnw$D2Fsut#Xf-hO&6uA0n~a;a3!=_!Tq^TdGE&<*c?1b|PovU}3tfiIUu z){4W|@PY}zJOXkGviCw^x27%K_Fm9GuKVpd{P2>NJlnk^I|h2XW0IO~LTMj>2<;S* zZh2uRNSdJM$U$@=`zz}%;ucRx{aKVxxF7?0hdKh6&GxO6f`l2kFncS3xu0Ly{ew0& zeEP*#lk-8-B$LD(5yj>YFJ{yf5zb41PlW7S{D9zC4Aa4nVdkDNH{UsFJp)q-`9OYt zbOKkigbmm5hF?tttn;S4g^142AF^`kiLUC?e7=*JH%Qe>uW=dB24NQa`;lm5yL>Dyh@HbHy-f%6Vz^ zh&MgwYsh(z#_fhhqY$3*f>Ha}*^cU-r4uTHaT?)~LUj5``FcS46oyoI5F3ZRizVD% zPFY(_S&5GN8$Nl2=+YO6j4d|M6O7CmUyS&}m4LSn6}J`$M0ZzT&Ome)ZbJDFvM&}A zZdhDn(*viM-JHf84$!I(8eakl#zRjJH4qfw8=60 z11Ely^FyXjVvtv48-Fae7p=adlt9_F^j5#ZDf7)n!#j?{W?@j$Pi=k`>Ii>XxrJ?$ z^bhh|X6qC8d{NS4rX5P!%jXy=>(P+r9?W(2)|(=a^s^l~x*^$Enw$~u%WRuRHHFan{X|S;FD(Mr z@r@h^@Bs#C3G;~IJMrERd+D!o?HmFX&#i|~q(7QR3f8QDip?ms6|GV_$86aDb|5pc?_-jo6vmWqYi{P#?{m_AesA4xX zi&ki&lh0yvf*Yw~@jt|r-=zpj!bw<6zI3Aa^Wq{|*WEC}I=O!Re!l~&8|Vu<$yZ1p zs-SlwJD8K!$(WWyhZ+sOqa8cciwvyh%zd`r$u;;fsHn!hub0VU)bUv^QH?x30#;tH zTc_VbZj|prj7)d%ORU;Vs{#ERb>K8>GOLSImnF7JhR|g$7FQTU{(a7RHQ*ii-{U3X z^7+vM0R$8b3k1aSU&kxvVPfOz3~)0O2iTYinV9_5{pF18j4b{o`=@AZIOAwwedB2@ ztXI1F04mg{<>a-gdFoRjq$6#FaevDn$^06L)k%wYq03&ysdXE+LL1#w$rRS1Y;BoS zH1x}{ms>LHWmdtP(ydD!aRdAa(d@csEo z0EF9L>%tppp`CZ2)jVb8AuoYyu;d^wfje6^n6`A?6$&%$p>HcE_De-Zh)%3o5)LDa zskQ}%o7?bg$xUj|n8gN9YB)z!N&-K&!_hVQ?#SFj+MpQA4@4oq!UQ$Vm3B`W_Pq3J z=ngFP4h_y=`Iar<`EESF9){%YZVyJqLPGq07TP7&fSDmnYs2NZQKiR%>){imTBJth zPHr@p>8b+N@~%43rSeNuOz;rgEm?14hNtI|KC6Xz1d?|2J`QS#`OW7gTF_;TPPxu@ z)9J9>3Lx*bc>Ielg|F3cou$O0+<b34_*ZJhpS&$8DP>s%47a)4ZLw`|>s=P_J4u z?I_%AvR_z8of@UYWJV?~c4Yb|A!9n!LEUE6{sn@9+D=0w_-`szJ_T++x3MN$v-)0d zy`?1QG}C^KiNlnJBRZBLr4G~15V3$QqC%1G5b#CEB0VTr#z?Ug%Jyv@a`QqAYUV~^ zw)d|%0g&kl{j#FMdf$cn(~L@8s~6eQ)6{`ik(RI(o9s0g30Li{4YoxcVoYd+LpeLz zai?~r)UcbYr@lv*Z>E%BsvTNd`Sc?}*}>mzJ|cr0Y(6rA7H_6&t>F{{mJ^xovc2a@ zFGGDUcGgI-z6H#o@Gj29C=Uy{wv zQHY2`HZu8+sBQK*_~I-_>fOTKEAQ8_Q~YE$c?cSCxI;vs-JGO`RS464Ft06rpjn+a zqRS0Y3oN(9HCP@{J4mOWqIyD8PirA!pgU^Ne{LHBG;S*bZpx3|JyQDGO&(;Im8!ed zNdpE&?3U?E@O~>`@B;oY>#?gXEDl3pE@J30R1;?QNNxZ?YePc)3=NS>!STCrXu*lM z69WkLB_RBwb1^-zEm*tkcHz3H;?v z;q+x0Jg$|?5;e1-kbJnuT+^$bWnYc~1qnyVTKh*cvM+8yJT-HBs1X@cD;L$su65;i z2c1MxyL~NuZ9+)hF=^-#;dS#lFy^Idcb>AEDXu1!G4Kd8YPy~0lZz$2gbv?su}Zn} zGtIbeYz3X8OA9{sT(aleold_?UEV{hWRl(@)NH6GFH@$<8hUt=dNte%e#Jc>7u9xi zuqv!CRE@!fmZZ}3&@$D>p0z=*dfQ_=IE4bG0hLmT@OP>x$e`qaqf_=#baJ8XPtOpWi%$ep1Y)o2(sR=v)M zt(z*pGS$Z#j_xq_lnCr+x9fwiT?h{NEn#iK(o)G&Xw-#DK?=Ms6T;%&EE${Gq_%99 z6(;P~jPKq9llc+cmI(MKQ6*7PcL)BmoI}MYFO)b3-{j>9FhNdXLR<^mnMP`I7z0v` zj3wxcXAqi4Z0kpeSf>?V_+D}NULgU$DBvZ^=0G8Bypd7P2>;u`yW9`%4~&tzNJpgp zqB+iLIM~IkB;ts!)exn643mAJ8-WlgFE%Rpq!UMYtB?$5QAMm)%PT0$$2{>Yu7&U@ zh}gD^Qdgu){y3ANdB5{75P;lRxSJPSpQPMJOiwmpMdT|?=q;&$aTt|dl~kvS z+*i;6cEQJ1V`R4Fd>-Uzsc=DPQ7A7#VPCIf!R!KK%LM&G%MoZ0{-8&99H!|UW$Ejv zhDLX3ESS6CgWTm#1ZeS2HJb`=UM^gsQ84dQpX(ESWSkjn>O zVxg%`@mh(X9&&wN$lDIc*@>rf?C0AD_mge3f2KkT6kGySOhXqZjtA?5z`vKl_{(5g z&%Y~9p?_DL{+q@siT~*3Q*$nWXQfNN;%s_eHP_A;O`N`SaoB z6xYR;z_;HQ2xAa9xKgx~2f2xEKiEDpGPH1d@||v#f#_Ty6_gY>^oZ#xac?pc-F`@ z*}8sPV@xiz?efDMcmmezYVw~qw=vT;G1xh+xRVBkmN66!u(mRG3G6P#v|;w@anEh7 zCf94arw%YB*=&3=RTqX?z4mID$W*^+&d6qI*LA-yGme;F9+wTsNXNaX~zl2+qIK&D-aeN4lr0+yP;W>|Dh?ms_ogT{DT+ ztXFy*R7j4IX;w@@R9Oct5k2M%&j=c_rWvoul+` z<18FH5D@i$P38W9VU2(EnEvlJ(SHCqTNBa)brkIjGP|jCnK&Qi%97tikU}Y#3L?s! z2ujL%YiHO-#!|g5066V01hgT#>fzls7P>+%D~ogOT&!Whb4iF=CnCto82Yb#b`YoVsj zS2q^W0Rj!RrM@=_GuPQy5*_X@Zmu`TKSbqEOP@;Ga&Rrr>#H@L41@ZX)LAkbo{G8+ z;!5EH6vv-ip0`tLB)xUuOX(*YEDSWf?PIxXe`+_B8=KH#HFCfthu}QJylPMTNmoV; zC63g%?57(&osaH^sxCyI-+gwVB|Xs2TOf=mgUAq?V~N_5!4A=b{AXbDae+yABuuu3B_XSa4~c z1s-OW>!cIkjwJf4ZhvT|*IKaRTU)WAK=G|H#B5#NB9<{*kt?7`+G*-^<)7$Iup@Um z7u*ABkG3F*Foj)W9-I&@BrN8(#$7Hdi`BU#SR1Uz4rh&=Ey!b76Qo?RqBJ!U+rh(1 znw@xw5$)4D8OWtB_^pJO*d~2Mb-f~>I!U#*=Eh*xa6$LX?4Evp4%;ENQR!mF4`f7F zpG!NX=qnCwE8@NAbQV`*?!v0;NJ(| zBip8}VgFVsXFqslXUV>_Z>1gmD(7p#=WACXaB|Y`=Kxa=p@_ALsL&yAJ`*QW^`2@% zW7~Yp(Q@ihmkf{vMF?kqkY%SwG^t&CtfRWZ{syK@W$#DzegcQ1>~r7foTw3^V1)f2Tq_5f$igmfch;8 zT-<)?RKcCdQh6x^mMEOS;4IpQ@F2q-4IC4%*dU@jfHR4UdG>Usw4;7ESpORL|2^#jd+@zxz{(|RV*1WKrw-)ln*8LnxVkKDfGDHA%7`HaiuvhMu%*mY9*Ya{Ti#{DW?i0 zXXsp+Bb(_~wv(3t70QU3a$*<$1&zm1t++x#wDLCRI4K)kU?Vm9n2c0m@TyUV&&l9%}fulj!Z9)&@yIcQ3gX}l0b1LbIh4S z5C*IDrYxR%qm4LVzSk{0;*npO_SocYWbkAjA6(^IAwUnoAzw_Uo}xYFo?Y<-4Zqec z&k7HtVlFGyt_pA&kX%P8PaRD8y!Wsnv}NMLNLy-CHZf(ObmzV|t-iC#@Z9*d-zUsx zxcYWw{H)nYXVdnJu5o-U+fn~W z-$h1ax>h{NlWLA7;;6TcQHA>UJB$KNk74T1xNWh9)kwK~wX0m|Jo_Z;g;>^E4-k4R zRj#pQb-Hg&dAh}*=2;JY*aiNZzT=IU&v|lQY%Q|=^V5pvTR7^t9+@+ST&sr!J1Y9a z514dYZn5rg6@4Cy6P`-?!3Y& z?B*5zw!mTiD2)>f@3XYrW^9V-@%YFkE_;PCyCJ7*?_3cR%tHng9%ZpIU}LJM=a+0s z(SDDLvcVa~b9O!cVL8)Q{d^R^(bbG=Ia$)dVN_tGMee3PMssZ7Z;c^Vg_1CjZYTnq z)wnF8?=-MmqVOMX!iE?YDvHCN?%TQtKJMFHp$~kX4}jZ;EDqP$?jqJZjoa2PM@$uZ zF4}iab1b5ep)L;jdegC3{K4VnCH#OV;pRcSa(&Nm50ze-yZ8*cGv;@+N+A?ncc^2z9~|(xFhwOHmPW@ zR5&)E^YKQj@`g=;zJ_+CLamsPuvppUr$G1#9urUj+p-mPW_QSSHkPMS!52t>Hqy|g z_@Yu3z%|wE=uYq8G>4`Q!4zivS}+}{m5Zjr7kMRGn_p&hNf|pc&f9iQ`^%78rl#~8 z;os@rpMA{ZioY~(Rm!Wf#Wx##A0PthOI341QiJ=G*#}pDAkDm+{0kz&*NB?rC0-)glB{0_Tq*^o zVS1>3REsv*Qb;qg!G^9;VoK)P*?f<*H&4Su1=}bP^Y<2PwFpoqw#up4IgX3L z`w~8jsFCI3k~Y9g(Y9Km`y$0FS5vHb)kb)Jb6q-9MbO{Hbb zxg?IWQ1ZIGgE}wKm{axO6CCh~4DyoFU+i1xn#oyfe+<{>=^B5tm!!*1M?AW8c=6g+%2Ft97_Hq&ZmOGvqGQ!Bn<_Vw`0DRuDoB6q8ME<;oL4kocr8E$NGoLI zXWmI7Af-DR|KJw!vKp2SI4W*x%A%5BgDu%8%Iato+pWo5`vH@!XqC!yK}KLzvfS(q z{!y(S-PKbk!qHsgVyxKsQWk_8HUSSmslUA9nWOjkKn0%cwn%yxnkfxn?Y2rysXKS=t-TeI%DN$sQ{lcD!(s>(4y#CSxZ4R} zFDI^HPC_l?uh_)-^ppeYRkPTPu~V^0Mt}#jrTL1Q(M;qVt4zb(L|J~sxx7Lva9`mh zz!#A9tA*6?q)xThc7(gB2Ryam$YG4qlh00c}r&$y6u zIN#Qxn{7RKJ+_r|1G1KEv!&uKfXpOVZ8tK{M775ws%nDyoZ?bi3NufNbZs)zqXiqc zqOsK@^OnlFMAT&mO3`@3nZP$3lLF;ds|;Z{W(Q-STa2>;)tjhR17OD|G>Q#zJHb*> zMO<{WIgB%_4MG0SQi2;%f0J8l_FH)Lfaa>*GLobD#AeMttYh4Yfg22@q4|Itq};NB z8;o*+@APqy@fPgrc&PTbGEwdEK=(x5K!If@R$NiO^7{#j9{~w=RBG)ZkbOw@$7Nhl zyp{*&QoVBd5lo{iwl2gfyip@}IirZK;ia(&ozNl!-EEYc=QpYH_= zJkv7gA{!n4up6$CrzDJIBAdC7D5D<_VLH*;OYN>_Dx3AT`K4Wyx8Tm{I+xplKP6k7 z2sb!i7)~%R#J0$|hK?~=u~rnH7HCUpsQJujDDE*GD`qrWWog+C+E~GGy|Hp_t4--} zrxtrgnPh}r=9o}P6jpAQuDN}I*GI`8&%Lp-C0IOJt#op)}XSr!ova@w{jG2V=?GXl3zEJJFXg)U3N>BQP z*Lb@%Mx|Tu;|u>$-K(q^-HG!EQ3o93%w(A7@ngGU)HRWoO&&^}U$5x+T&#zri>6ct zXOB#EF-;z3j311K`jrYyv6pOPF=*`SOz!ack=DuEi({UnAkL5H)@R?YbRKAeP|06U z?-Ns0ZxD0h9D8)P66Sq$w-yF+1hEVTaul%&=kKDrQtF<$RnQPZ)ezm1`aHIjAY=!S z`%vboP`?7mItgEo4w50C*}Ycqp9_3ZEr^F1;cEhkb`BNhbc6PvnXu@wi=AoezF4~K zkxx%ps<8zb=wJ+9I8o#do)&{(=yAlNdduaDn!=xGSiuo~fLw~Edw$6;l-qaq#Z7?# zGrdU(Cf-V@$x>O%yRc6!C1Vf`b19ly;=mEu8u9|zitcG^O`lbNh}k=$%a)UHhDwTEKis2yc4rBGR>l*(B$AC7ung&ssaZGkY-h(fpwcPyJSx*9EIJMRKbMP9}$nVrh6$g-Q^5Cw)BeWqb-qi#37ZXKL!GR;ql)~ z@PP*-oP?T|ThqlGKR84zi^CN z4TZ1A)7vL>ivoL2EU_~xl-P{p+sE}9CRwGJDKy{>0KP+gj`H9C+4fUMPnIB1_D`A- z$1`G}g0lQmqMN{Y&8R*$xYUB*V}dQPxGVZQ+rH!DVohIoTbh%#z#Tru%Px@C<=|og zGDDwGq7yz`%^?r~6t&>x*^We^tZ4!E4dhwsht#Pb1kCY{q#Kv;z%Dp#Dq;$vH$-(9 z8S5tutZ}&JM2Iw&Y-7KY4h5BBvS=Ove0#+H2qPdR)WyI zYcj)vB=MA{7T|3Ij_PN@FM@w(C9ANBq&|NoW30ccr~i#)EcH)T^3St~rJ0HKKd4wr z@_+132;Bj+>UC@h)Ap*8B4r5A1lZ!Dh%H7&&hBnlFj@eayk=VD*i5AQc z$uN8YG#PL;cuQa)Hyt-}R?&NAE1QT>svJDKt*)AQOZAJ@ zyxJoBebiobHeFlcLwu_iI&NEZuipnOR;Tn;PbT1Mt-#5v5b*8ULo7m)L-eti=UcGf zRZXidmxeFgY!y80-*PH-*=(-W+fK%KyUKpg$X@tuv``tXj^*4qq@UkW$ZrAo%+hay zU@a?z&2_@y)o@D!_g>NVxFBO!EyB&6Z!nd4=KyDP^hl!*(k{dEF6@NkXztO7gIh zQ&PC+p-8WBv;N(rpfKdF^@Z~|E6pa)M1NBUrCZvLRW$%N%xIbv^uv?=C!=dDVq3%* zgvbEBnG*JB*@vXx8>)7XL*!{1Jh=#2UrByF7U?Rj_}VYw88BwqefT_cCTv8aTrRVjnn z1HNCF=44?*&gs2`vCGJVHX@kO z240eo#z+FhI0=yy6NHQwZs}a+J~4U-6X`@ zZ7j+tb##m`x%J66$a9qXDHG&^kp|GkFFMmjD(Y-k_ClY~N$H|n@NkSDz=gg?*2ga5 z)+f)MEY>2Lp15;~o`t`qj;S>BaE;%dv@Ux11yq}I(k|o&`5UZFUHn}1kE^gIK@qV& z!S2IhyU;->VfA4Qb}m7YnkIa9%z{l~iPWo2YPk-`hy2-Eg=6E$21plQA5W2qMZDFU z-a-@Dndf%#on6chT`dOKnU9}BJo|kJwgGC<^nfo34zOKH96LbWY7@Wc%EoFF=}`VU zksP@wd%@W;-p!e^&-)N7#oR331Q)@9cx=mOoU?_Kih2!Le*8fhsZ8Qvo6t2vt+UOZ zw|mCB*t2%z21YqL>whu!j?s~}-L`OS+jdg1(XnmYw$rg~r(?5Y+qTg`$F}q3J?GtL z@BN&8#`u2RqkdG4yGGTus@7U_%{6C{XAhFE!2SelH?KtMtX@B1GBhEIDL-Bj#~{4! zd}p7!#XE9Lt;sy@p5#Wj*jf8zGv6tTotCR2X$EVOOup;GnRPRVU5A6N@Lh8?eA7k? zn~hz&gY;B0ybSpF?qwQ|sv_yO=8}zeg2$0n3A8KpE@q26)?707pPw?H76lCpjp=5r z6jjp|auXJDnW}uLb6d7rsxekbET9(=zdTqC8(F5@NNqII2+~yB;X5iJNQSiv`#ozm zf&p!;>8xAlwoxUC3DQ#!31ylK%VrcwS<$WeCY4V63V!|221oj+5#r}fGFQ}|uwC0) zNl8(CF}PD`&Sj+p{d!B&&JtC+VuH z#>US`)YQrhb6lIAYb08H22y(?)&L8MIQsA{26X`R5Km{YU)s!x(&gIsjDvq63@X`{ z=7{SiH*_ZsPME#t2m|bS76Uz*z{cpp1m|s}HIX}Ntx#v7Eo!1%G9__4dGSGl`p+xi zZ!VK#Qe;Re=9bqXuW+0DSP{uZ5-QXrNn-7qW19K0qU}OhVru7}3vqsG?#D67 zb}crN;QwsH*vymw(maZr_o|w&@sQki(X+D)gc5Bt&@iXisFG;eH@5d43~Wxq|HO(@ zV-rip4n#PEkHCWCa5d?@cQp^B;I-PzOfag|t-cuvTapQ@MWLmh*41NH`<+A+JGyKX zyYL6Ba7qqa5j@3lOk~`OMO7f0!@FaOeZxkbG@vXP(t3#U*fq8=GAPqUAS>vW2uxMk{a(<0=IxB;# zMW;M+owrHaZBp`3{e@7gJCHP!I(EeyGFF;pdFPdeP+KphrulPSVidmg#!@W`GpD&d z9p6R`dpjaR2E1Eg)Ws{BVCBU9-aCgN57N~uLvQZH`@T+2eOBD%73rr&sV~m#2~IZx zY_8f8O;XLu2~E3JDXnGhFvsyb^>*!D>5EtlKPe%kOLv6*@=Jpci`8h0z?+fbBUg_7 zu6DjqO=$SjAv{|Om5)nz41ZkS4E_|fk%NDY509VV5yNeo%O|sb>7C#wj8mL9cEOFh z>nDz%?vb!h*!0dHdnxDA>97~EoT~!N40>+)G2CeYdOvJr5^VnkGz)et&T9hrD(VAgCAJjQ7V$O?csICB*HFd^k@$M5*v$PZJD-OVL?Ze(U=XGqZPVG8JQ z<~ukO%&%nNXYaaRibq#B1KfW4+XMliC*Tng2G(T1VvP;2K~;b$EAqthc${gjn_P!b zs62UT(->A>!ot}cJXMZHuy)^qfqW~xO-In2);e>Ta{LD6VG2u&UT&a@>r-;4<)cJ9 zjpQThb4^CY)Ev0KR7TBuT#-v}W?Xzj{c7$S5_zJA57Qf=$4^npEjl9clH0=jWO8sX z3Fuu0@S!WY>0XX7arjH`?)I<%2|8HfL!~#c+&!ZVmhbh`wbzy0Ux|Jpy9A{_7GGB0 zadZ48dW0oUwUAHl%|E-Q{gA{z6TXsvU#Hj09<7i)d}wa+Iya)S$CVwG{4LqtB>w%S zKZx(QbV7J9pYt`W4+0~f{hoo5ZG<0O&&5L57oF%hc0xGJ@Zrg_D&lNO=-I^0y#3mxCSZFxN2-tN_mU@7<@PnWG?L5OSqkm8TR!`| zRcTeWH~0z1JY^%!N<(TtxSP5^G9*Vw1wub`tC-F`=U)&sJVfvmh#Pi`*44kSdG};1 zJbHOmy4Ot|%_?@$N?RA9fF?|CywR8Sf(SCN_luM8>(u0NSEbKUy7C(Sk&OuWffj)f za`+mo+kM_8OLuCUiA*CNE|?jra$M=$F3t+h-)?pXz&r^F!ck;r##`)i)t?AWq-9A9 zSY{m~TC1w>HdEaiR*%j)L);H{IULw)uxDO>#+WcBUe^HU)~L|9#0D<*Ld459xTyew zbh5vCg$a>`RCVk)#~ByCv@Ce!nm<#EW|9j><#jQ8JfTmK#~jJ&o0Fs9jz0Ux{svdM4__<1 zrb>H(qBO;v(pXPf5_?XDq!*3KW^4>(XTo=6O2MJdM^N4IIcYn1sZZpnmMAEdt}4SU zPO54j2d|(xJtQ9EX-YrlXU1}6*h{zjn`in-N!Ls}IJsG@X&lfycsoCemt_Ym(PXhv zc*QTnkNIV=Ia%tg%pwJtT^+`v8ng>;2~ps~wdqZSNI7+}-3r+#r6p`8*G;~bVFzg= z!S3&y)#iNSUF6z;%o)%h!ORhE?CUs%g(k2a-d576uOP2@QwG-6LT*G!I$JQLpd`cz z-2=Brr_+z96a0*aIhY2%0(Sz=|D`_v_7h%Yqbw2)8@1DwH4s*A82krEk{ zoa`LbCdS)R?egRWNeHV8KJG0Ypy!#}kslun?67}^+J&02!D??lN~t@;h?GS8#WX`)6yC**~5YNhN_Hj}YG<%2ao^bpD8RpgV|V|GQwlL27B zEuah|)%m1s8C6>FLY0DFe9Ob66fo&b8%iUN=y_Qj;t3WGlNqP9^d#75ftCPA*R4E8 z)SWKBKkEzTr4JqRMEs`)0;x8C35yRAV++n(Cm5++?WB@ya=l8pFL`N0ag`lWhrYo3 zJJ$< zQ*_YAqIGR*;`VzAEx1Pd4b3_oWtdcs7LU2#1#Ls>Ynvd8k^M{Ef?8`RxA3!Th-?ui{_WJvhzY4FiPxA?E4+NFmaC-Uh*a zeLKkkECqy>Qx&1xxEhh8SzMML=8VP}?b*sgT9ypBLF)Zh#w&JzP>ymrM?nnvt!@$2 zh>N$Q>mbPAC2kNd&ab;FkBJ}39s*TYY0=@e?N7GX>wqaM>P=Y12lciUmve_jMF0lY zBfI3U2{33vWo(DiSOc}!5##TDr|dgX1Uojq9!vW3$m#zM_83EGsP6&O`@v-PDdO3P z>#!BEbqpOXd5s?QNnN!p+92SHy{sdpePXHL{d@c6UilT<#~I!tH$S(~o}c#(j<2%! zQvm}MvAj-95Ekx3D4+|e%!?lO(F+DFw9bxb-}rsWQl)b44###eUg4N?N-P(sFH2hF z`{zu?LmAxn2=2wCE8?;%ZDi#Y;Fzp+RnY8fWlzVz_*PDO6?Je&aEmuS>=uCXgdP6r zoc_JB^TA~rU5*geh{G*gl%_HnISMS~^@{@KVC;(aL^ZA-De+1zwUSXgT>OY)W?d6~ z72znET0m`53q%AVUcGraYxIcAB?OZA8AT!uK8jU+=t;WneL~|IeQ>$*dWa#x%rB(+ z5?xEkZ&b{HsZ4Ju9TQ|)c_SIp`7r2qMJgaglfSBHhl)QO1aNtkGr0LUn{@mvAt=}nd7#>7ru}&I)FNsa*x?Oe3-4G`HcaR zJ}c%iKlwh`x)yX1vBB;-Nr=7>$~(u=AuPX2#&Eh~IeFw%afU+U)td0KC!pHd zyn+X$L|(H3uNit-bpn7%G%{&LsAaEfEsD?yM<;U2}WtD4KuVKuX=ec9X zIe*ibp1?$gPL7<0uj*vmj2lWKe`U(f9E{KVbr&q*RsO;O>K{i-7W)8KG5~~uS++56 zm@XGrX@x+lGEjDQJp~XCkEyJG5Y57omJhGN{^2z5lj-()PVR&wWnDk2M?n_TYR(gM zw4kQ|+i}3z6YZq8gVUN}KiYre^sL{ynS}o{z$s&I z{(rWaLXxcQ=MB(Cz7W$??Tn*$1y(7XX)tv;I-{7F$fPB%6YC7>-Dk#=Y8o1=&|>t5 zV_VVts>Eb@)&4%m}!K*WfLoLl|3FW)V~E1Z!yu`Sn+bAP5sRDyu7NEbLt?khAyz-ZyL-}MYb&nQ zU16f@q7E1rh!)d%f^tTHE3cVoa%Xs%rKFc|temN1sa)aSlT*)*4k?Z>b3NP(IRXfq zlB^#G6BDA1%t9^Nw1BD>lBV(0XW5c?l%vyB3)q*;Z5V~SU;HkN;1kA3Nx!$!9wti= zB8>n`gt;VlBt%5xmDxjfl0>`K$fTU-C6_Z;!A_liu0@Os5reMLNk;jrlVF^FbLETI zW+Z_5m|ozNBn7AaQ<&7zk}(jmEdCsPgmo%^GXo>YYt82n&7I-uQ%A;k{nS~VYGDTn zlr3}HbWQG6xu8+bFu^9%%^PYCbkLf=*J|hr>Sw+#l(Y#ZGKDufa#f-f0k-{-XOb4i zwVG1Oa0L2+&(u$S7TvedS<1m45*>a~5tuOZ;3x%!f``{=2QQlJk|b4>NpD4&L+xI+ z+}S(m3}|8|Vv(KYAGyZK5x*sgwOOJklN0jsq|BomM>OuRDVFf_?cMq%B*iQ*&|vS9 zVH7Kh)SjrCBv+FYAE=$0V&NIW=xP>d-s7@wM*sdfjVx6-Y@=~>rz%2L*rKp|*WXIz z*vR^4tV&7MQpS9%{9b*>E9d_ls|toL7J|;srnW{l-}1gP_Qr-bBHt=}PL@WlE|&KH zCUmDLZb%J$ZzNii-5VeygOM?K8e$EcK=z-hIk63o4y63^_*RdaitO^THC{boKstphXZ2Z+&3ToeLQUG(0Frs?b zCxB+65h7R$+LsbmL51Kc)pz_`YpGEzFEclzb=?FJ=>rJwgcp0QH-UuKRS1*yCHsO) z-8t?Zw|6t($Eh&4K+u$I7HqVJBOOFCRcmMMH};RX_b?;rnk`rz@vxT_&|6V@q0~Uk z9ax|!pA@Lwn8h7syrEtDluZ6G!;@=GL> zse#PRQrdDs=qa_v@{Wv(3YjYD0|qocDC;-F~&{oaTP?@pi$n z1L6SlmFU2~%)M^$@C(^cD!y)-2SeHo3t?u3JiN7UBa7E2 z;<+_A$V084@>&u)*C<4h7jw9joHuSpVsy8GZVT;(>lZ(RAr!;)bwM~o__Gm~exd`K zKEgh2)w?ReH&syI`~;Uo4`x4$&X+dYKI{e`dS~bQuS|p zA`P_{QLV3r$*~lb=9vR^H0AxK9_+dmHX}Y} zIV*#65%jRWem5Z($ji{!6ug$En4O*=^CiG=K zp4S?+xE|6!cn$A%XutqNEgUqYY3fw&N(Z6=@W6*bxdp~i_yz5VcgSj=lf-6X1Nz75 z^DabwZ4*70$$8NsEy@U^W67tcy7^lNbu;|kOLcJ40A%J#pZe0d#n zC{)}+p+?8*ftUlxJE*!%$`h~|KZSaCb=jpK3byAcuHk7wk@?YxkT1!|r({P*KY^`u z!hw#`5$JJZGt@nkBK_nwWA31_Q9UGvv9r-{NU<&7HHMQsq=sn@O?e~fwl20tnSBG* zO%4?Ew6`aX=I5lqmy&OkmtU}bH-+zvJ_CFy z_nw#!8Rap5Wcex#5}Ldtqhr_Z$}@jPuYljTosS1+WG+TxZ>dGeT)?ZP3#3>sf#KOG z0)s%{cEHBkS)019}-1A2kd*it>y65-C zh7J9zogM74?PU)0c0YavY7g~%j%yiWEGDb+;Ew5g5Gq@MpVFFBNOpu0x)>Yn>G6uo zKE%z1EhkG_N5$a8f6SRm(25iH#FMeaJ1^TBcBy<04ID47(1(D)q}g=_6#^V@yI?Y&@HUf z`;ojGDdsvRCoTmasXndENqfWkOw=#cV-9*QClpI03)FWcx(m5(P1DW+2-{Hr-`5M{v##Zu-i-9Cvt;V|n)1pR^y ztp3IXzHjYWqabuPqnCY9^^;adc!a%Z35VN~TzwAxq{NU&Kp35m?fw_^D{wzB}4FVXX5Zk@#={6jRh%wx|!eu@Xp;%x+{2;}!&J4X*_SvtkqE#KDIPPn@ z5BE$3uRlb>N<2A$g_cuRQM1T#5ra9u2x9pQuqF1l2#N{Q!jVJ<>HlLeVW|fN|#vqSnRr<0 zTVs=)7d`=EsJXkZLJgv~9JB&ay16xDG6v(J2eZy;U%a@EbAB-=C?PpA9@}?_Yfb&) zBpsih5m1U9Px<+2$TBJ@7s9HW>W){i&XKLZ_{1Wzh-o!l5_S+f$j^RNYo85}uVhN# zq}_mN-d=n{>fZD2Lx$Twd2)}X2ceasu91}n&BS+4U9=Y{aZCgV5# z?z_Hq-knIbgIpnkGzJz-NW*=p?3l(}y3(aPCW=A({g9CpjJfYuZ%#Tz81Y)al?!S~ z9AS5#&nzm*NF?2tCR#|D-EjBWifFR=da6hW^PHTl&km-WI9*F4o>5J{LBSieVk`KO z2(^9R(zC$@g|i3}`mK-qFZ33PD34jd_qOAFj29687wCUy>;(Hwo%Me&c=~)V$ua)V zsaM(aThQ3{TiM~;gTckp)LFvN?%TlO-;$y+YX4i`SU0hbm<})t0zZ!t1=wY&j#N>q zONEHIB^RW6D5N*cq6^+?T}$3m|L{Fe+L!rxJ=KRjlJS~|z-&CC{#CU8`}2|lo~)<| zk?Wi1;Cr;`?02-C_3^gD{|Ryhw!8i?yx5i0v5?p)9wZxSkwn z3C;pz25KR&7{|rc4H)V~y8%+6lX&KN&=^$Wqu+}}n{Y~K4XpI-#O?L=(2qncYNePX zTsB6_3`7q&e0K67=Kg7G=j#?r!j0S^w7;0?CJbB3_C4_8X*Q%F1%cmB{g%XE&|IA7 z(#?AeG{l)s_orNJp!$Q~qGrj*YnuKlV`nVdg4vkTNS~w$4d^Oc3(dxi(W5jq0e>x} z(GN1?u2%Sy;GA|B%Sk)ukr#v*UJU%(BE9X54!&KL9A^&rR%v zIdYt0&D59ggM}CKWyxGS@ z>T#})2Bk8sZMGJYFJtc>D#k0+Rrrs)2DG;(u(DB_v-sVg=GFMlSCx<&RL;BH}d6AG3VqP!JpC0Gv6f8d|+7YRC@g|=N=C2 zo>^0CE0*RW?W))S(N)}NKA)aSwsR{1*rs$(cZIs?nF9)G*bSr%%SZo^YQ|TSz={jX z4Z+(~v_>RH0(|IZ-_D_h@~p_i%k^XEi+CJVC~B zsPir zA0Jm2yIdo4`&I`hd%$Bv=Rq#-#bh{Mxb_{PN%trcf(#J3S1UKDfC1QjH2E;>wUf5= ze8tY9QSYx0J;$JUR-0ar6fuiQTCQP#P|WEq;Ez|*@d?JHu-(?*tTpGHC+=Q%H>&I> z*jC7%nJIy+HeoURWN%3X47UUusY2h7nckRxh8-)J61Zvn@j-uPA@99|y48pO)0XcW zX^d&kW^p7xsvdX?2QZ8cEUbMZ7`&n{%Bo*xgFr4&fd#tHOEboQos~xm8q&W;fqrj} z%KYnnE%R`=`+?lu-O+J9r@+$%YnqYq!SVs>xp;%Q8p^$wA~oynhnvIFp^)Z2CvcyC zIN-_3EUHW}1^VQ0;Oj>q?mkPx$Wj-i7QoXgQ!HyRh6Gj8p~gH22k&nmEqUR^)9qni{%uNeV{&0-H60C zibHZtbV=8=aX!xFvkO}T@lJ_4&ki$d+0ns3FXb+iP-VAVN`B7f-hO)jyh#4#_$XG%Txk6M<+q6D~ zi*UcgRBOoP$7P6RmaPZ2%MG}CMfs=>*~(b97V4+2qdwvwA@>U3QQAA$hiN9zi%Mq{ z*#fH57zUmi)GEefh7@`Uy7?@@=BL7cXbd{O9)*lJh*v!@ z-6}p9u0AreiGauxn7JBEa-2w&d=!*TLJ49`U@D7%2ppIh)ynMaAE2Q4dl@47cNu{9 z&3vT#pG$#%hrXzXsj=&Ss*0;W`Jo^mcy4*L8b^sSi;H{*`zW9xX2HAtQ*sO|x$c6UbRA(7*9=;D~(%wfo(Z6#s$S zuFk`dr%DfVX5KC|Af8@AIr8@OAVj=6iX!~8D_P>p7>s!Hj+X0_t}Y*T4L5V->A@Zx zcm1wN;TNq=h`5W&>z5cNA99U1lY6+!!u$ib|41VMcJk8`+kP{PEOUvc@2@fW(bh5pp6>C3T55@XlpsAd#vn~__3H;Dz2w=t9v&{v*)1m4)vX;4 zX4YAjM66?Z7kD@XX{e`f1t_ZvYyi*puSNhVPq%jeyBteaOHo7vOr8!qqp7wV;)%jtD5>}-a?xavZ;i|2P3~7c)vP2O#Fb`Y&Kce zQNr7%fr4#S)OOV-1piOf7NgQvR{lcvZ*SNbLMq(olrdDC6su;ubp5un!&oT=jVTC3uTw7|r;@&y*s)a<{J zkzG(PApmMCpMmuh6GkM_`AsBE@t~)EDcq1AJ~N@7bqyW_i!mtHGnVgBA`Dxi^P93i z5R;}AQ60wy=Q2GUnSwz+W6C^}qn`S-lY7=J(3#BlOK%pCl=|RVWhC|IDj1E#+|M{TV0vE;vMZLy7KpD1$Yk zi0!9%qy8>CyrcRK`juQ)I};r)5|_<<9x)32b3DT1M`>v^ld!yabX6@ihf`3ZVTgME zfy(l-ocFuZ(L&OM4=1N#Mrrm_<>1DZpoWTO70U8+x4r3BpqH6z@(4~sqv!A9_L}@7 z7o~;|?~s-b?ud&Wx6==9{4uTcS|0-p@dKi0y#tPm2`A!^o3fZ8Uidxq|uz2vxf;wr zM^%#9)h^R&T;}cxVI(XX7kKPEVb);AQO?cFT-ub=%lZPwxefymBk+!H!W(o(>I{jW z$h;xuNUr#^0ivvSB-YEbUqe$GLSGrU$B3q28&oA55l)ChKOrwiTyI~e*uN;^V@g-Dm4d|MK!ol8hoaSB%iOQ#i_@`EYK_9ZEjFZ8Ho7P^er z^2U6ZNQ{*hcEm?R-lK)pD_r(e=Jfe?5VkJ$2~Oq^7YjE^5(6a6Il--j@6dBHx2Ulq z!%hz{d-S~i9Eo~WvQYDt7O7*G9CP#nrKE#DtIEbe_uxptcCSmYZMqT2F}7Kw0AWWC zPjwo0IYZ6klc(h9uL|NY$;{SGm4R8Bt^^q{e#foMxfCSY^-c&IVPl|A_ru!ebwR#7 z3<4+nZL(mEsU}O9e`^XB4^*m)73hd04HH%6ok^!;4|JAENnEr~%s6W~8KWD)3MD*+ zRc46yo<}8|!|yW-+KulE86aB_T4pDgL$XyiRW(OOcnP4|2;v!m2fB7Hw-IkY#wYfF zP4w;k-RInWr4fbz=X$J;z2E8pvAuy9kLJUSl8_USi;rW`kZGF?*Ur%%(t$^{Rg!=v zg;h3@!Q$eTa7S0#APEDHLvK%RCn^o0u!xC1Y0Jg!Baht*a4mmKHy~88md{YmN#x) zBOAp_i-z2h#V~*oO-9k(BizR^l#Vm%uSa^~3337d;f=AhVp?heJ)nlZGm`}D(U^2w z#vC}o1g1h?RAV^90N|Jd@M00PoNUPyA?@HeX0P7`TKSA=*4s@R;Ulo4Ih{W^CD{c8 ze(ipN{CAXP(KHJ7UvpOc@9SUAS^wKo3h-}BDZu}-qjdNlVtp^Z{|CxKOEo?tB}-4; zEXyDzGbXttJ3V$lLo-D?HYwZm7vvwdRo}P#KVF>F|M&eJ44n*ZO~0)#0e0Vy&j00I z{%IrnUvKp70P?>~J^$^0Wo%>le>re2ZSvRfes@dC-*e=DD1-j%<$^~4^4>Id5w^Fr z{RWL>EbUCcyC%1980kOYqZAcgdz5cS8c^7%vvrc@CSPIx;X=RuodO2dxk17|am?HJ@d~Mp_l8H?T;5l0&WGFoTKM{eP!L-a0O8?w zgBPhY78tqf^+xv4#OK2I#0L-cSbEUWH2z+sDur85*!hjEhFfD!i0Eyr-RRLFEm5(n z-RV6Zf_qMxN5S6#8fr9vDL01PxzHr7wgOn%0Htmvk9*gP^Um=n^+7GLs#GmU&a#U^4jr)BkIubQO7oUG!4CneO2Ixa`e~+Jp9m{l6apL8SOqA^ zvrfEUPwnHQ8;yBt!&(hAwASmL?Axitiqvx%KZRRP?tj2521wyxN3ZD9buj4e;2y6U zw=TKh$4%tt(eh|y#*{flUJ5t4VyP*@3af`hyY^YU3LCE3Z|22iRK7M7E;1SZVHbXF zKVw!L?2bS|kl7rN4(*4h2qxyLjWG0vR@`M~QFPsf^KParmCX;Gh4OX6Uy9#4e_%oK zv1DRnfvd$pu(kUoV(MmAc09ckDiuqS$a%!AQ1Z>@DM#}-yAP$l`oV`BDYpkqpk(I|+qk!yoo$TwWr6dRzLy(c zi+qbVlYGz0XUq@;Fm3r~_p%by)S&SVWS+wS0rC9bk^3K^_@6N5|2rtF)wI>WJ=;Fz zn8$h<|Dr%kN|nciMwJAv;_%3XG9sDnO@i&pKVNEfziH_gxKy{l zo`2m4rnUT(qenuq9B0<#Iy(RPxP8R)=5~9wBku=%&EBoZ82x1GlV<>R=hIqf0PK!V zw?{z9e^B`bGyg2nH!^x}06oE%J_JLk)^QyHLipoCs2MWIqc>vaxsJj(=gg1ZSa=u{ zt}od#V;e7sA4S(V9^<^TZ#InyVBFT(V#$fvI7Q+pgsr_2X`N~8)IOZtX}e(Bn(;eF zsNj#qOF_bHl$nw5!ULY{lNx@93Fj}%R@lewUuJ*X*1$K`DNAFpE z7_lPE+!}uZ6c?+6NY1!QREg#iFy=Z!OEW}CXBd~wW|r_9%zkUPR0A3m+@Nk%4p>)F zXVut7$aOZ6`w}%+WV$te6-IX7g2yms@aLygaTlIv3=Jl#Nr}nN zp|vH-3L03#%-1-!mY`1z?+K1E>8K09G~JcxfS)%DZbteGQnQhaCGE2Y<{ut#(k-DL zh&5PLpi9x3$HM82dS!M?(Z zEsqW?dx-K_GMQu5K54pYJD=5+Rn&@bGjB?3$xgYl-|`FElp}?zP&RAd<522c$Rv6} zcM%rYClU%JB#GuS>FNb{P2q*oHy}UcQ-pZ2UlT~zXt5*k-ZalE(`p7<`0n7i(r2k{ zb84&^LA7+aW1Gx5!wK!xTbw0slM?6-i32CaOcLC2B>ZRI16d{&-$QBEu1fKF0dVU>GTP05x2>Tmdy`75Qx! z^IG;HB9V1-D5&&)zjJ&~G}VU1-x7EUlT3QgNT<&eIDUPYey$M|RD6%mVkoDe|;2`8Z+_{0&scCq>Mh3hj|E*|W3;y@{$qhu77D)QJ` znD9C1AHCKSAHQqdWBiP`-cAjq7`V%~JFES1=i-s5h6xVT<50kiAH_dn0KQB4t*=ua zz}F@mcKjhB;^7ka@WbSJFZRPeYI&JFkpJ-!B z!ju#!6IzJ;D@$Qhvz9IGY5!%TD&(db3<*sCpZ?U#1^9RWQ zs*O-)j!E85SMKtoZzE^8{w%E0R0b2lwwSJ%@E}Lou)iLmPQyO=eirG8h#o&E4~eew z;h><=|4m0$`ANTOixHQOGpksXlF0yy17E&JksB4_(vKR5s$Ve+i;gco2}^RRJI+~R zWJ82WGigLIUwP!uSELh3AAs9HmY-kz=_EL-w|9}noKE#(a;QBpEx9 z4BT-zY=6dJT>72Hkz=9J1E=}*MC;zzzUWb@x(Ho8cU_aRZ?fxse5_Ru2YOvcr?kg&pt@v;{ai7G--k$LQtoYj+Wjk+nnZty;XzANsrhoH#7=xVqfPIW(p zX5{YF+5=k4_LBnhLUZxX*O?29olfPS?u*ybhM_y z*XHUqM6OLB#lyTB`v<BZ&YRs$N)S@5Kn_b3;gjz6>fh@^j%y2-ya({>Hd@kv{CZZ2e)tva7gxLLp z`HoGW);eRtov~Ro5tetU2y72~ zQh>D`@dt@s^csdfN-*U&o*)i3c4oBufCa0e|BwT2y%Y~=U7A^ny}tx zHwA>Wm|!SCko~UN?hporyQHRUWl3djIc722EKbTIXQ6>>iC!x+cq^sUxVSj~u)dsY zW8QgfZlE*2Os%=K;_vy3wx{0u!2%A)qEG-$R^`($%AOfnA^LpkB_}Dd7AymC)zSQr z>C&N8V57)aeX8ap!|7vWaK6=-3~ko9meugAlBKYGOjc#36+KJwQKRNa_`W@7;a>ot zdRiJkz?+QgC$b}-Owzuaw3zBVLEugOp6UeMHAKo2$m4w zpw?i%Lft^UtuLI}wd4(-9Z^*lVoa}11~+0|Hs6zAgJ01`dEA&^>Ai=mr0nC%eBd_B zzgv2G_~1c1wr*q@QqVW*Wi1zn=}KCtSwLjwT>ndXE_Xa22HHL_xCDhkM( zhbw+j4uZM|r&3h=Z#YrxGo}GX`)AZyv@7#7+nd-D?BZV>thtc|3jt30j$9{aIw9)v zDY)*fsSLPQTNa&>UL^RWH(vpNXT7HBv@9=*=(Q?3#H*crA2>KYx7Ab?-(HU~a275)MBp~`P)hhzSsbj|d`aBe(L*(;zif{iFJu**ZR zkL-tPyh!#*r-JVQJq>5b0?cCy!uSKef+R=$s3iA7*k*_l&*e!$F zYwGI;=S^0)b`mP8&Ry@{R(dPfykD&?H)na^ihVS7KXkxb36TbGm%X1!QSmbV9^#>A z-%X>wljnTMU0#d;tpw?O1W@{X-k*>aOImeG z#N^x?ehaaQd}ReQykp>i;92q@%$a!y1PNyPYDIvMm& zyYVwn;+0({W@3h(r&i#FuCDE)AC(y&Vu>4?1@j0|CWnhHUx4|zL7cdaA32RSk?wl% zMK^n42@i5AU>f70(huWfOwaucbaToxj%+)7hnG^CjH|O`A}+GHZyQ-X57(WuiyRXV zPf>0N3GJ<2Myg!sE4XJY?Z7@K3ZgHy8f7CS5ton0Eq)Cp`iLROAglnsiEXpnI+S8; zZn>g2VqLxi^p8#F#Laf3<00AcT}Qh&kQnd^28u!9l1m^`lfh9+5$VNv=?(~Gl2wAl zx(w$Z2!_oESg_3Kk0hUsBJ<;OTPyL(?z6xj6LG5|Ic4II*P+_=ac7KRJZ`(k2R$L# zv|oWM@116K7r3^EL*j2ktjEEOY9c!IhnyqD&oy7+645^+@z5Y|;0+dyR2X6^%7GD* zXrbPqTO}O={ z4cGaI#DdpP;5u?lcNb($V`l>H7k7otl_jQFu1hh>=(?CTPN#IPO%O_rlVX}_Nq;L< z@YNiY>-W~&E@=EC5%o_z<^3YEw)i_c|NXxHF{=7U7Ev&C`c^0Z4-LGKXu*Hkk&Av= zG&RAv{cR7o4${k~f{F~J48Ks&o(D@j-PQ2`LL@I~b=ifx3q!p6`d>~Y!<-^mMk3)e zhi1;(YLU5KH}zzZNhl^`0HT(r`5FfmDEzxa zk&J7WQ|!v~TyDWdXQ)!AN_Y%xM*!jv^`s)A`|F%;eGg27KYsrCE2H}7*r)zvum6B{ z$k5Har9pv!dcG%f|3hE(#hFH+12RZPycVi?2y`-9I7JHryMn3 z9Y8?==_(vOAJ7PnT<0&85`_jMD0#ipta~Q3M!q5H1D@Nj-YXI$W%OQplM(GWZ5Lpq z-He6ul|3<;ZQsqs!{Y7x`FV@pOQc4|N;)qgtRe(Uf?|YqZv^$k8On7DJ5>f2%M=TV zw~x}9o=mh$JVF{v4H5Su1pq66+mhTG6?F>Do}x{V(TgFwuLfvNP^ijkrp5#s4UT!~ zEU7pr8aA)2z1zb|X9IpmJykQcqI#(rS|A4&=TtWu@g^;JCN`2kL}%+K!KlgC z>P)v+uCeI{1KZpewf>C=?N7%1e10Y3pQCZST1GT5fVyB1`q)JqCLXM zSN0qlreH1=%Zg-5`(dlfSHI&2?^SQdbEE&W4#%Eve2-EnX>NfboD<2l((>>34lE%) zS6PWibEvuBG7)KQo_`?KHSPk+2P;`}#xEs}0!;yPaTrR#j(2H|#-CbVnTt_?9aG`o z(4IPU*n>`cw2V~HM#O`Z^bv|cK|K};buJ|#{reT8R)f+P2<3$0YGh!lqx3&a_wi2Q zN^U|U$w4NP!Z>5|O)>$GjS5wqL3T8jTn%Vfg3_KnyUM{M`?bm)9oqZP&1w1)o=@+(5eUF@=P~ zk2B5AKxQ96n-6lyjh&xD!gHCzD$}OOdKQQk7LXS-fk2uy#h{ktqDo{o&>O!6%B|)` zg?|JgcH{P*5SoE3(}QyGc=@hqlB5w;bnmF#pL4iH`TSuft$dE5j^qP2S)?)@pjRQZ zBfo6g>c!|bN-Y|(Wah2o61Vd|OtXS?1`Fu&mFZ^yzUd4lgu7V|MRdGj3e#V`=mnk- zZ@LHn?@dDi=I^}R?}mZwduik!hC%=Hcl56u{Wrk1|1SxlgnzG&e7Vzh*wNM(6Y!~m z`cm8Ygc1$@z9u9=m5vs1(XXvH;q16fxyX4&e5dP-{!Kd555FD6G^sOXHyaCLka|8j zKKW^E>}>URx736WWNf?U6Dbd37Va3wQkiE;5F!quSnVKnmaIRl)b5rM_ICu4txs+w zj}nsd0I_VG^<%DMR8Zf}vh}kk;heOQTbl ziEoE;9@FBIfR7OO9y4Pwyz02OeA$n)mESpj zdd=xPwA`nO06uGGsXr4n>Cjot7m^~2X~V4yH&- zv2llS{|und45}Pm1-_W@)a-`vFBpD~>eVP(-rVHIIA|HD@%7>k8JPI-O*<7X{L*Ik zh^K`aEN!BteiRaY82FVo6<^8_22=aDIa8P&2A3V<(BQ;;x8Zs-1WuLRWjQvKv1rd2 zt%+fZ!L|ISVKT?$3iCK#7whp|1ivz1rV*R>yc5dS3kIKy_0`)n*%bfNyw%e7Uo}Mnnf>QwDgeH$X5eg_)!pI4EJjh6?kkG2oc6Af0py z(txE}$ukD|Zn=c+R`Oq;m~CSY{ebu9?!is}01sOK_mB?{lSY33E=!KkKtMeI*FO2b z%95awv9;Z|UDp3xm+aP*5I!R-_M2;GxeCRx3ATS0iF<_Do2Mi)Hk2 zjBF35VB>(oamIYjunu?g0O-?LuOvtfs5F(iiIicbu$HMPPF%F>pE@hIRjzT)>aa=m zwe;H9&+2|S!m74!E3xfO{l3E_ab`Q^tZ4yH9=~o2DUEtEMDqG=&D*8!>?2uao%w`&)THr z^>=L3HJquY>6)>dW4pCWbzrIB+>rdr{s}}cL_?#!sOPztRwPm1B=!jP7lQG|Iy6rP zVqZDNA;xaUx&xUt?Ox|;`9?oz`C0#}mc<1Urs#vTW4wd{1_r`eX=BeSV z_9WV*9mz>PH6b^z{VYQJ1nSTSqOFHE9u>cY)m`Q>=w1NzUShxcHsAxasnF2BG;NQ; zqL1tjLjImz_`q=|bAOr_i5_NEijqYZ^;d5y3ZFj6kCYakJh**N_wbfH;ICXq?-p#r z{{ljNDPSytOaG#7=yPmA&5gyYI%^7pLnMOw-RK}#*dk=@usL;|4US?{@K%7esmc&n z5$D*+l&C9)Bo@$d;Nwipd!68&+NnOj^<~vRcKLX>e03E|;to;$ndgR;9~&S-ly5gf z{rzj+j-g$;O|u?;wwxrEpD=8iFzUHQfl{B>bLHqH(9P zI59SS2PEBE;{zJUlcmf(T4DrcO?XRWR}?fekN<($1&AJTRDyW+D*2(Gyi?Qx-i}gy z&BpIO!NeVdLReO!YgdUfnT}7?5Z#~t5rMWqG+$N2n%5o#Np6ccNly}#IZQsW4?|NV zR9hrcyP(l#A+U4XcQvT;4{#i)dU>HK>aS!k1<3s2LyAhm2(!Nu%vRC9T`_yn9D+r} z1i&U~IcQ?4xhZYyH6WL-f%}qIhZkc&}n2N0PM| z6|XA9d-y;!`D{p;xu*gv7a|zaZ*MiQ)}zPzW4GB0mr)}N-DmB&hl1&x`2@sxN572_ zS)RdJyR%<7kW0v3Q_|57JKy&9tUdbqz}|hwn84}U*0r^jt6Ssrp+#1y=JBcZ+F`f(N?O0XL1OFGN`1-r?S<#t4*C9|y~e)!UYZ zRQ3M8m%~M)VriIvn~XzoP;5qeu(ZI>Y#r zAd)J)G9)*BeE%gmm&M@Olg3DI_zokjh9NvdGbT z+u4(Y&uC6tBBefIg~e=J#8i1Zxr>RT)#rGaB2C71usdsT=}mm`<#WY^6V{L*J6v&l z1^Tkr6-+^PA)yC;s1O^3Q!)Reb=fxs)P~I*?i&j{Vbb(Juc?La;cA5(H7#FKIj0Or zgV0BO{DUs`I9HgQ{-!g@5P^Vr|C4}~w6b=#`Zx0XcVSd?(04HUHwK(gJNafgQNB9Z zCi3TgNXAeJ+x|X|b@27$RxuYYuNSUBqo#uyiH6H(b~K*#!@g__4i%HP5wb<+Q7GSb zTZjJw96htUaGZ89$K_iBo4xEOJ#DT#KRu9ozu!GH0cqR>hP$nk=KXM%Y!(%vWQ#}s zy=O#BZ>xjUejMH^F39Bf0}>D}yiAh^toa-ts#gt6Mk9h1D<9_mGMBhLT0Ce2O3d_U znaTkBaxd-8XgwSp5)x-pqX5=+{cSuk6kyl@k|5DQ!5zLUVV%1X9vjY0gerbuG6nwZu5KDMdq(&UMLZ zy?jW#F6joUtVyz`Y?-#Yc0=i*htOFwQ3`hk$8oq35D}0m$FAOp#UFTV3|U3F>@N?d zeXLZCZjRC($%?dz(41e~)CN10qjh^1CdAcY(<=GMGk@`b1ptA&L*{L@_M{%Vd5b*x#b1(qh=7((<_l%ZUaHtmgq} zjchBdiis{Afxf@3CjPR09E*2#X(`W#-n`~6PcbaL_(^3tfDLk?Nb6CkW9v!v#&pWJ3iV-9hz zngp#Q`w`r~2wt&cQ9#S7z0CA^>Mzm7fpt72g<0y-KT{G~l-@L#edmjZQ}7{*$mLgSdJfS$Ge{hrD=mr;GD)uYq8}xS zT>(w_;}894Kb}(P5~FOpFIEjadhmxD(PsZbKwa-qxVa7Oc7~ebPKMeN(pCRzq8s@l z`|l^*X1eK1+Spz--WkSW_nK`Cs@JmkY4+p=U91nJoy{tSH;TzuIyS)Q_(S@;Iakua zpuDo5W54Mo;jY@Ly1dY)j|+M%$FJ0`C=FW#%UvOd&?p}0QqL20Xt!#pr8ujy6CA-2 zFz6Ex5H1i)c9&HUNwG{8K%FRK7HL$RJwvGakleLLo}tsb>t_nBCIuABNo$G--_j!gV&t8L^4N6wC|aLC)l&w04CD6Vc#h^(YH@Zs4nwUGkhc_-yt{dK zMZ<%$swLmUl8`E~RLihGt@J5v;r;vT&*Q!Cx zZ55-zpb;W7_Q{tf$mQvF61(K>kwTq0x{#Din||)B{+6O#ArLi)kiHWVC4`fOT&B(h zw&YV`J1|^FLx~9Q%r-SFhYl4PywI7sF2Q$>4o50~dfp5nn}XHv-_DM?RGs#+4gM;% znU>k=81G~f6u%^Z{bcX&sUv*h|L+|mNq=W43y@{~C zpL-TW3hYPs0^*OqS#KQwA^CGG_A-6#`_{1LBCD&*3nY0UHWJj1D|VP%oQlFxLllaA zVI@2^)HZ%E*=RbQcFOKIP7?+|_xVK+2oG(t_EGl2y;Ovox zZb^qVpe!4^reKvpIBFzx;Ji=PmrV>uu-Hb>`s?k?YZQ?>av45>i(w0V!|n?AP|v5H zm`e&Tgli#lqGEt?=(?~fy<(%#nDU`O@}Vjib6^rfE2xn;qgU6{u36j_+Km%v*2RLnGpsvS+THbZ>p(B zgb{QvqE?~50pkLP^0(`~K& zjT=2Pt2nSnwmnDFi2>;*C|OM1dY|CAZ5R|%SAuU|5KkjRM!LW_)LC*A zf{f>XaD+;rl6Y>Umr>M8y>lF+=nSxZX_-Z7lkTXyuZ(O6?UHw^q; z&$Zsm4U~}KLWz8>_{p*WQ!OgxT1JC&B&>|+LE3Z2mFNTUho<0u?@r^d=2 z-av!n8r#5M|F%l;=D=S1mGLjgFsiYAOODAR}#e^a8 zfVt$k=_o}kt3PTz?EpLkt54dY}kyd$rU zVqc9SN>0c z753j-gdN~UiW*FUDMOpYEkVzP)}{Ds*3_)ZBi)4v26MQr140|QRqhFoP=a|;C{#KS zD^9b-9HM11W+cb1Y)HAuk<^GUUo(ut!5kILBzAe)Vaxwu4Up!7Ql*#DDu z>EB84&xSrh>0jT!*X81jJQq$CRHqNj29!V3FN9DCx)~bvZbLwSlo3l^zPb1sqBnp) zfZpo|amY^H*I==3#8D%x3>zh#_SBf?r2QrD(Y@El!wa;Ja6G9Y1947P*DC|{9~nO& z*vDnnU!8(cV%HevsraF%Y%2{Z>CL0?64eu9r^t#WjW4~3uw8d}WHzsV%oq-T)Y z0-c!FWX5j1{1##?{aTeCW2b$PEnwe;t`VPCm@sQ`+$$L2=3kBR%2XU1{_|__XJ$xt zibjY2QlDVs)RgHH*kl&+jn*JqquF)k_Ypibo00lcc<2RYqsi-G%}k0r(N97H7JEn7@E3ZTH0JK>d8)E~A-D z!B&z9zJw0Bi^fgQZI%LirYaBKnWBXgc`An*qvO^*$xymqKOp(+3}IsnVhu?YnN7qz zNJxDN-JWd7-vIiv2M9ih>x3gNVY%DzzY~dCnA}76IRl!`VM=6=TYQ=o&uuE8kHqZT zoUNod0v+s9D)7aLJ|hVqL0li1hg)%&MAciI(4YJ=%D4H$fGQ&Lu-?@>>@pEgC;ERrL= zI^cS&3q8fvEGTJZgZwL5j&jp%j9U^Of6pR{wA^u=tVt#yCQepXNIbynGnuWbsC_EE zRyMFq{5DK692-*kyGy~An>AdVR9u___fzmmJ4;^s0yAGgO^h{YFmqJ%ZJ_^0BgCET zE6(B*SzeZ4pAxear^B-YW<%BK->X&Cr`g9_;qH~pCle# zdY|UB5cS<}DFRMO;&czbmV(?vzikf)Ks`d$LL801@HTP5@r><}$xp}+Ip`u_AZ~!K zT}{+R9Wkj}DtC=4QIqJok5(~0Ll&_6PPVQ`hZ+2iX1H{YjI8axG_Bw#QJy`6T>1Nn z%u^l`>XJ{^vX`L0 z1%w-ie!dE|!SP<>#c%ma9)8K4gm=!inHn2U+GR+~ zqZVoa!#aS0SP(|**WfQSe?cA=1|Jwk`UDsny%_y{@AV??N>xWekf>_IZLUEK3{Ksi zWWW$if&Go~@Oz)`#=6t_bNtD$d9FMBN#&97+XKa+K2C@I9xWgTE{?Xnhc9_KKPcujj@NprM@e|KtV_SR+ zSpeJ!1FGJ=Te6={;;+;a46-*DW*FjTnBfeuzI_=I1yk8M(}IwEIGWV0Y~wia;}^dg z{BK#G7^J`SE10z4(_Me=kF&4ld*}wpNs91%2Ute>Om`byv9qgK4VfwPj$`axsiZ)wxS4k4KTLb-d~!7I@^Jq`>?TrixHk|9 zqCX7@sWcVfNP8N;(T>>PJgsklQ#GF>F;fz_Rogh3r!dy*0qMr#>hvSua;$d z3TCZ4tlkyWPTD<=5&*bUck~J;oaIzSQ0E03_2x{?weax^jL3o`ZP#uvK{Z5^%H4b6 z%Kbp6K?>{;8>BnQy64Jy$~DN?l(ufkcs6TpaO&i~dC>0fvi-I^7YT#h?m;TVG|nba%CKRG%}3P*wejg) zI(ow&(5X3HR_xk{jrnkA-hbwxEQh|$CET9Qv6UpM+-bY?E!XVorBvHoU59;q<9$hK z%w5K-SK zWT#1OX__$ceoq0cRt>9|)v}$7{PlfwN}%Wh3rwSl;%JD|k~@IBMd5}JD#TOvp=S57 zae=J#0%+oH`-Av}a(Jqhd4h5~eG5ASOD)DfuqujI6p!;xF_GFcc;hZ9k^a7c%%h(J zhY;n&SyJWxju<+r`;pmAAWJmHDs{)V-x7(0-;E?I9FWK@Z6G+?7Py8uLc2~Fh1^0K zzC*V#P88(6U$XBjLmnahi2C!a+|4a)5Ho5>owQw$jaBm<)H2fR=-B*AI8G@@P-8I8 zHios92Q6Nk-n0;;c|WV$Q);Hu4;+y%C@3alP`cJ2{z~*m-@de%OKVgiWp;4Q)qf9n zJ!vmx(C=_>{+??w{U^Bh|LFJ<6t}Er<-Tu{C{dv8eb(kVQ4!fOuopTo!^x1OrG}0D zR{A#SrmN`=7T29bzQ}bwX8OUufW9d9T4>WY2n15=k3_rfGOp6sK0oj7(0xGaEe+-C zVuWa;hS*MB{^$=0`bWF(h|{}?53{5Wf!1M%YxVw}io4u-G2AYN|FdmhI13HvnoK zNS2fStm=?8ZpKt}v1@Dmz0FD(9pu}N@aDG3BY8y`O*xFsSz9f+Y({hFx;P_h>ER_& z`~{z?_vCNS>agYZI?ry*V96_uh;|EFc0*-x*`$f4A$*==p`TUVG;YDO+I4{gJGrj^ zn?ud(B4BlQr;NN?vaz_7{&(D9mfd z8esj=a4tR-ybJjCMtqV8>zn`r{0g$hwoWRUI3}X5=dofN){;vNoftEwX>2t@nUJro z#%7rpie2eH1sRa9i6TbBA4hLE8SBK@blOs=ouBvk{zFCYn4xY;v3QSM%y6?_+FGDn z4A;m)W?JL!gw^*tRx$gqmBXk&VU=Nh$gYp+Swu!h!+e(26(6*3Q!(!MsrMiLri`S= zKItik^R9g!0q7y$lh+L4zBc-?Fsm8`CX1+f>4GK7^X2#*H|oK}reQnT{Mm|0ar<+S zRc_dM%M?a3bC2ILD`|;6vKA`a3*N~(cjw~Xy`zhuY2s{(7KLB{S>QtR3NBQ3>vd+= z#}Q)AJr7Y_-eV(sMN#x!uGX08oE*g=grB*|bBs}%^3!RVA4f%m3=1f0K=T^}iI&2K zuM2GG5_%+#v-&V>?x4W9wQ|jE2Q7Be8mOyJtZrqn#gXy-1fF1P$C8+We&B*-pi#q5 zETp%H6g+%#sH+L4=ww?-h;MRCd2J9zwQUe4gHAbCbH08gDJY;F6F)HtWCRW1fLR;)ysGZanlz*a+|V&@(ipWdB!tz=m_0 z6F}`d$r%33bw?G*azn*}Z;UMr{z4d9j~s`0*foZkUPwpJsGgoR0aF>&@DC;$A&(av z?b|oo;`_jd>_5nye`DVOcMLr-*Nw&nA z82E8Dw^$Lpso)gEMh?N|Uc^X*NIhg=U%enuzZOGi-xcZRUZmkmq~(cP{S|*+A6P;Q zprIkJkIl51@ng)8cR6QSXJtoa$AzT@*(zN3M+6`BTO~ZMo0`9$s;pg0HE3C;&;D@q zd^0zcpT+jC%&=cYJF+j&uzX87d(gP9&kB9|-zN=69ymQS9_K@h3ph&wD5_!4q@qI@ zBMbd`2JJ2%yNX?`3(u&+nUUJLZ=|{t7^Rpw#v-pqD2_3}UEz!QazhRty%|Q~WCo7$ z+sIugHA%Lmm{lBP#bnu_>G}Ja<*6YOvSC;89z67M%iG0dagOt1HDpDn$<&H0DWxMU zxOYaaks6%R@{`l~zlZ*~2}n53mn2|O&gE+j*^ypbrtBv{xd~G(NF?Z%F3>S6+qcry z?ZdF9R*a;3lqX_!rI(Cov8ER_mOqSn6g&ZU(I|DHo7Jj`GJ}mF;T(vax`2+B8)H_D zD0I;%I?*oGD616DsC#j0x*p+ZpBfd=9gR|TvB)832CRhsW_7g&WI@zp@r7dhg}{+4f=(cO2s+)jg0x(*6|^+6W_=YIfSH0lTcK* z%)LyaOL6em@*-_u)}Swe8rU)~#zT-vNiW(D*~?Zp3NWl1y#fo!3sK-5Ek6F$F5l3| zrFFD~WHz1}WHmzzZ!n&O8rTgfytJG*7iE~0`0;HGXgWTgx@2fD`oodipOM*MOWN-} zJY-^>VMEi8v23ZlOn0NXp{7!QV3F1FY_URZjRKMcY(2PV_ms}EIC^x z=EYB5UUQ{@R~$2Mwiw$_JAcF+szKB*n(`MYpDCl>~ss54uDQ%Xf-8|dgO zY)B_qju=IaShS|XsQo=nSYxV$_vQR@hd~;qW)TEfU|BA0&-JSwO}-a*T;^}l;MgLM zz}CjPlJX|W2vCzm3oHw3vqsRc3RY=2()}iw_k2#eKf&VEP7TQ;(DDzEAUgj!z_h2Br;Z3u=K~LqM6YOrlh)v9`!n|6M-s z?XvA~y<5?WJ{+yM~uPh7uVM&g-(;IC3>uA}ud?B3F zelSyc)Nx>(?F=H88O&_70%{ATsLVTAp88F-`+|egQ7C4rpIgOf;1tU1au+D3 zlz?k$jJtTOrl&B2%}D}8d=+$NINOZjY$lb{O<;oT<zXoAp01KYG$Y4*=)!&4g|FL(!54OhR-?)DXC&VS5E|1HGk8LY;)FRJqnz zb_rV2F7=BGwHgDK&4J3{%&IK~rQx<&Kea|qEre;%A~5YD6x`mo>mdR)l?Nd%T2(5U z_ciT02-zt_*C|vn?BYDuqSFrk3R(4B0M@CRFmG{5sovIq4%8AhjXA5UwRGo)MxZlI zI%vz`v8B+#ff*XtGnciczFG}l(I}{YuCco#2E6|+5WJ|>BSDfz0oT+F z%QI^ixD|^(AN`MS6J$ zXlKNTFhb>KDkJp*4*LaZ2WWA5YR~{`={F^hwXGG*rJYQA7kx|nwnC58!eogSIvy{F zm1C#9@$LhK^Tl>&iM0wsnbG7Y^MnQ=q))MgApj4)DQt!Q5S`h+5a%c7M!m%)?+h65 z0NHDiEM^`W+M4)=q^#sk(g!GTpB}edwIe>FJQ+jAbCo#b zXmtd3raGJNH8vnqMtjem<_)9`gU_-RF&ZK!aIenv7B2Y0rZhon=2yh&VsHzM|`y|0x$Zez$bUg5Nqj?@~^ zPN43MB}q0kF&^=#3C;2T*bDBTyO(+#nZnULkVy0JcGJ36or7yl1wt7HI_>V7>mdud zv2II9P61FyEXZuF$=69dn%Z6F;SOwyGL4D5mKfW)q4l$8yUhv7|>>h_-4T*_CwAyu7;DW}_H zo>N_7Gm6eed=UaiEp_7aZko@CC61@(E1be&5I9TUq%AOJW>s^9w%pR5g2{7HW9qyF zh+ZvX;5}PN0!B4q2FUy+C#w5J?0Tkd&S#~94(AP4%fRb^742pgH7Tb1))siXWXHUT z1Wn5CG&!mGtr#jq6(P#!ck@K+FNprcWP?^wA2>mHA03W?kj>5b|P0ErXS) zg2qDTjQ|grCgYhrH-RapWCvMq5vCaF?{R%*mu}1)UDll~6;}3Q*^QOfj!dlt02lSzK z?+P)02Rrq``NbU3j&s*;<%i4Y>y9NK&=&KsYwvEmf5jwTG6?+Pu1q9M8lLlx)uZZ7 zizhr~e0ktGs-=$li-2jz^_48-jk**y&5u0`B2gc#i$T1~t+AS*kEfR*b{^Ec>2-F~ zKYRl&uQ5yO@EtAZX8ZSqx;8+AKf+CqhlUSpp*VfyBMv+%wxN5GukZEi^_to%MFRc0 zdXqJ*jk?#uYT6EJe446@(f6G4vhnxQP|pGeJ?-#|Ksq?g*ky=}x+Qnx+!<>Y(XStN zQIND`{KU}&l)E*ntI^}kJ=ly8DML{!(58Xk4_bzIc@v~e;>wKl_`7G%pGz~4KH*CTp;_|52)d!+ximd$|8v@zzEq%j68QXkgf$7eM~xdM5q5i z{?qFx_W|eq@L03bWJfjy^z@()-iCjzjREuf zb_a(yTz)ZKWCF%Lp>^2-%Q?*t{06}x#DLN3cO=i>h6#-a`z;<5rBGGM6GA(WqvRcX%Pn?Uvs1#e|ePSNJEC%+X(YI$x)`s$%>O#%}D9dgqWfq4yfVz^%FglokdFR}uJQhx|}_w`9Ulx38Ha>ZslKs58c-@IFI&f;?xM zbK>rKNfPFsf>%+k6%(A6=7Aac^_qrOCNqb3ZVJ;8pt!?1DR*ynJb#@II9h?)xB)A~ zm9Kk)Hy}!Z+W}i6ZJDy+?yY_=#kWrzgV)2eZAx_E=}Nh7*#<&mQz`Umfe$+l^P(xd zN}PA2qII4}ddCU+PN+yxkH%y!Qe(;iH3W%bwM3NKbU_saBo<8x9fGNtTAc_SizU=o zC3n2;c%LoU^j90Sz>B_p--Fzqv7x7*?|~-x{haH8RP)p|^u$}S9pD-}5;88pu0J~9 zj}EC`Q^Fw}`^pvAs4qOIuxKvGN@DUdRQ8p-RXh=3S#<`3{+Qv6&nEm)uV|kRVnu6f zco{(rJaWw(T0PWim?kkj9pJ)ZsUk9)dSNLDHf`y&@wbd;_ita>6RXFJ+8XC*-wsiN z(HR|9IF283fn=DI#3Ze&#y3yS5;!yoIBAH(v}3p5_Zr+F99*%+)cp!Sy8e+lG?dOc zuEz<;3X9Z5kkpL_ZYQa`sioR_@_cG z8tT~GOSTWnO~#?$u)AcaBSaV7P~RT?Nn8(OSL1RmzPWRWQ$K2`6*)+&7^zZBeWzud z*xb3|Fc~|R9eH+lQ#4wF#c;)Gka6lL(63C;>(bZob!i8F-3EhYU3|6-JBC0*5`y0| zBs!Frs=s!Sy0qmQNgIH|F`6(SrD1js2prni_QbG9Sv@^Pu2szR9NZl8GU89gWWvVg z2^-b*t+F{Nt>v?js7hnlC`tRU(an0qQG7;h6T~ z-`vf#R-AE$pzk`M{gCaia}F`->O2)60AuGFAJg> z*O2IZqTx=AzDvC49?A92>bQLdb&32_4>0Bgp0ESXXnd4B)!$t$g{*FG%HYdt3b3a^J9#so%BJMyr2 z{y?rzW!>lr097b9(75#&4&@lkB1vT*w&0E>!dS+a|ZOu6t^zro2tiP)bhcNNxn zbJs3_Fz+?t;4bkd8GfDI7ccJ5zU`Bs~ zN~bci`c`a%DoCMel<-KUCBdZRmew`MbZEPYE|R#|*hhvhyhOL#9Yt7$g_)!X?fK^F z8UDz)(zpsvriJ5aro5>qy`Fnz%;IR$@Kg3Z3EE!fv9CAdrAym6QU82=_$_N5*({_1 z7!-=zy(R{xg9S519S6W{HpJZ8Is|kQ!0?`!vxDggmslD59)>iQ15f z7J8NqdR`9f8H|~iFGNsPV!N)(CC9JRmzL9S}7U-K@`X893f3f<8|8Ls!^eA^#(O6nA+ByFIXcz_WLbfeG|nHJ5_sJJ^gNJ%SI9#XEfNRbzV+!RkI zXS$MOVYb2!0vU}Gt7oUy*|WpF^*orBot~b2J@^be?Gq;U%#am8`PmH-UCFZ&uTJlnetYij0z{K1mmivk$bdPbLodu;-R@@#gAV!=d%(caz$E?r zURX0pqAn7UuF6dULnoF1dZ$WM)tHAM{eZK6DbU1J`V5Dw<;xk}Nl`h+nfMO_Rdv z3SyOMzAbYaD;mkxA7_I_DOs#Bk;e5D%gsS3q)hlmi1w{FsjKNJE22`AjmNiAPRnIc zcIkN25;rOn3FipAFd(PnlK9{03w6Q<(68#1Jw`{axEGQE{Ac>^U$h);h2ADICmaNxrfpb`Jdr*)Y1SicpYKCFv$3vf~;5aW>n^7QGa63MJ z;B1+Z>WQ615R2D8JmmT`T{QcgZ+Kz1hTu{9FOL}Q8+iFx-Vyi}ZVVcGjTe>QfA`7W zFoS__+;E_rQIQxd(Bq4$egKeKsk#-9=&A!)(|hBvydsr5ts0Zjp*%*C0lM2sIOx1s zg$xz?Fh?x!P^!vWa|}^+SY8oZHub7f;E!S&Q;F?dZmvBxuFEISC}$^B_x*N-xRRJh zn4W*ThEWaPD*$KBr8_?}XRhHY7h^U1aN6>m=n~?YJQd8+!Uyq_3^)~4>XjelM&!c9 zCo|0KsGq7!KsZ~9@%G?i>LaU7#uSTMpypocm*oqJHR|wOgVWc7_8PVuuw>x{kEG4T z$p^DV`}jUK39zqFc(d5;N+M!Zd3zhZN&?Ww(<@AV-&f!v$uV>%z+dg9((35o@4rqLvTC-se@hkn^6k7+xHiK-vTRvM8{bCejbU;1@U=*r}GTI?Oc$!b6NRcj83-zF; z=TB#ESDB`F`jf4)z=OS76Se}tQDDHh{VKJk#Ad6FDB_=afpK#pyRkGrk~OuzmQG)} z*$t!nZu$KN&B;|O-aD=H<|n6aGGJZ=K9QFLG0y=Jye_ElJFNZJT;fU8P8CZcLBERjioAOC0Vz_pIXIc};)8HjfPwNy zE!g|lkRv3qpmU?shz(BBt5%TbpJC3HzP9!t7k*Fh48!-HlJ4TTgdCr3rCU!iF}kgu z4Qs;K@XOY~4f~N}Jl8V_mGbwzvNLbl&0e9UG4W;kvjTK|5`-Ld+eQ6YRF`N0ct%u% z^3J_{7r#_W1zm|>IPN!yWCRrN)N!7v`~ptNkIXKipQ6ogFvcnI5ugxdoa{d;uD67g zgo^}QuZRkB540Vc!@c80(wFG=$ct}oHq(#W0+-XX(;Rrt`x=<45X}ficNtI2(&}=~ zb(!}tNz?s`wm{gK?2tdf+OEF;tzx<(3fMd7_tM@Ghs$Z(Os-H(kYq#qB|J-aC9Ku?fsWwJhB36c)A zu|a7ZF?V8X7l2g5~xqZf>2=6Dsi5lfo zKIRL&@MLJyaBE)V_9=pJYu%U2wxR*-(0MI5_|yqP`?h@cks(5LR@XUKLMI_xuVtiu zRvpDS8MyUMRFM6`P+Sjc!A_e^H38Qu7b{b7QZ>NHyA6k-YYygQuW&C_OGO(7V7?}r)zedSVpBI zuk29Z4GW3C0GpfozbZQya454sjt@ndQmsp=DA&@sWw&xmOlDk1JIcMNp~-ES$&A~k zG#W(6hBj?!Fu8Q4WYexoSBa8_5=v20xnx6H?e;$t)5|f&{7=vOye^&3_c-Ug?|a@e z=X`&qT_5B7N9vZoPBhXOTEDV;4&x2Je4}T(UB~O-$D#CjX77$R?RZ*`ed~$G;$4YS z4n*|Pop(!NN79Hk2}U#cfEEwdxM)xQm}$~rV03xc=#U@@Y*}qEmot5KvDb=8{!E-n zl4p?}&g2h^sUGyTcGh=0aQzQb*k;K;dvbeZUgmwEv>%#(EPtj=gHKdi|E8@w+|>KC zxEU>b>P+9Xf}pEyQK(}#QrBG4Jaf!iE!qpMbTu>gb!gtdq<`@xO+roQl+S_7)!G(% zdy)$iGmJ1cwP?F=IyyV1-$|kf|EKM3B@I&lZ%NI@VV;*mQdLWjc#t|Vbk_Q~>&O03 zIcSr$(qLAINj7a z;!||v&1D5SX#X@5jNd}jUsi-CH_Scjyht&}q2p*CJCC-`&NyXf)vD5{e!HO629D-O z%bZelTcq=DoRX>zeWCa^RmR3*{x9;3lZ75M#S)!W0bRIFH#P6b%{|HRSZ5!!I#s)W z_|XXZQ<0_`>b^^0Z>LU64Yg1w)8}#M^9se(OZ9~baZ7fsKFc;EtnB>kesci#>=icG zuHdjax2^=!_(9?0l7;G7^-}9>Y#M zm;9*GT~dBuYWdk49%mZM0=H#FY1)}7NE5DE_vsqrA0`?0R0q535qHjWXcl|gz9Fq$ zMKxgL;68l!gm3y0durIr3LHv~y*ABm` zYhQG0UW#hg@*A{&G!;$FS43}rIF$e6yRdGJWVR<}uuJ_5_8qa3xaHH^!VzUteVp;> z<0`M>3tnY$ZFb$(`0sg93TwGyP;`9UYUWxO&CvAnSzei&ap))NcW;R`tA=y^?mBmG+M*&bqW5kL$V(O;(p)aEk`^ci?2Jwxu>0sy>a7+Wa9t z5#I2o;+gr^9^&km^z7>xJWbN&Ft>Vna34E zI@BBzwX)R}K3SL?)enrDJ45QLt;-7CFJk{`cF3L4Z^CtG_r5)0)HV>BOYPIUh#D%| zYQAu31f{bm-D*`_k7DTTr?Nkw_gY%J1cb2&TdtibY?V=|SSIOlA;|5C!2@?YQ z-$?G0jj^mG|MP>DmbF7}T~C$H6=CpZ~hd zZ1C|xV@=h#^~`3LSCnmI(vZ|5r3>eq5*UB)dhdy``*gKY3Eg%jSK8I-`G+OWWlD)T zt$wSQ=||lSkiKy}YF-k}@W9EiS?)z`hK{R!dd-$BCJvBtAN-yXn3njU$MisEtp!?Q z%Vk-*(wy9dd15(-WFw_&^tT;;IpF?ox1`Qq3-0zVTk+$W_?q}GfAQlPcrB^?&tWSI z2BB!K=sH7FUYmXa_dcV^Z3>5z8}~W{S!$jVR_3hu_|wl2|gmRH8ftn^z@fW75*;-`;wU+fY+BR_yx6BZnE5_Hna({jrPiubRp$jZ=T=t$hx&NeCV1!vuCcl4PJ0p0Fjp>6K} zHkoD1gQk=P2hYcT%)cJ2Q5WuA|5_x+dX0%hnozfTF>$#Wz~X!MY>){H4#fB#7^ID* z1*o2Hzp}?WVs&gbS?Uq(CT0sP+F)u9{xfgg6o_{8J#m;|NeJqDHhb(Q8%z8aM_qeM zn83>d`uDd47WIuKp78JBYo2SYupGcNXIzeou^eMY`@%Bv8elZ>q~3uq#~IX)g%g;h zoUXymEd>|kVsMkyb&1l~lrE-`w(0PObapYa35DJ4Y03Jv_!DKp}0HTbOgZRM=;PSsuAJJJ1 zItc+tu9;ANG;qHaCI|T85!euhFK~VK^G2LZV1+cbzS?>ar@>emg;JTI5VAn1g5U~| zU=p&k0OlSzc$U=s#9_uL3&n|6A1X$XvrE9vFV@`A4G#!D1QcFCeE`F2N(deJx>)*A z$XIW0P~-NbAd=5i6`s<~(vAQX9t$dbVqc5|E|CHRtb$1(l&KSNh_t2#k_l95KnP86 z)ns_DGspv-M0z0#h2a+*oH|{5~j{ zXGD=}cLrBSESQ0u$XmQlFfWMCAWaS;wKK%#aSSYK=qljBiY(s zT$v;We24&$w=avIILsMt0%1fDyah|AlLNg#WL$Lu)tf}YfqO%+pH~QC*bZO4aM*i9 zrPFf|5!hv@XY8CzaFh*Dy9vH|2fKKr(@x}`L#9^*vOae|lk`adG#oZZAyk|TOV8`9L zc-sQu%y1MQes&J?)a1}Zc*>-P!6j-T#75V$lLC!TuMB(!G-+D2;XptUxymSPFI-K&0x}B1?h$ z3-9**-9!);fwyiWB5gS$i;P~c=^}5-6G@{4TWDBRDc6(M|%qa-mS`z`u9kWo{Xl_uc;hXOkRd diff --git a/flow-jvm/gradle/wrapper/gradle-wrapper.properties b/flow-jvm/gradle/wrapper/gradle-wrapper.properties index 17655d0ef2..a80b22ce5c 100644 --- a/flow-jvm/gradle/wrapper/gradle-wrapper.properties +++ b/flow-jvm/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/flow-jvm/gradlew b/flow-jvm/gradlew index fbd7c51583..1aa94a4269 100755 --- a/flow-jvm/gradlew +++ b/flow-jvm/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,99 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +119,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,88 +130,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/flow-jvm/gradlew.bat b/flow-jvm/gradlew.bat index a9f778a7a9..7101f8e467 100644 --- a/flow-jvm/gradlew.bat +++ b/flow-jvm/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,13 +41,13 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -54,31 +55,16 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line @@ -86,17 +72,19 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/flow-jvm/src/test/java/io/peerdb/IcebergServiceTest.java b/flow-jvm/src/test/java/io/peerdb/IcebergServiceTest.java deleted file mode 100644 index 29e8193f60..0000000000 --- a/flow-jvm/src/test/java/io/peerdb/IcebergServiceTest.java +++ /dev/null @@ -1,22 +0,0 @@ -//package io.peerdb; -// -//import io.peerdb.flow.jvm.iceberg.resource.IcebergResource; -//import io.peerdb.flow.jvm.iceberg.service.IcebergService; -//import io.quarkus.grpc.GrpcClient; -//import io.quarkus.test.junit.QuarkusTest; -//import org.junit.jupiter.api.Test; -// -//@QuarkusTest -//public class IcebergServiceTest { -// @GrpcClient -// IcebergService client; -// -// @Test -// public void printRecordCount() { -// var response = client.createTable(null).onFailure().invoke(e -> { -// System.out.println(e); -// }).onFailure().recoverWithNull().await().indefinitely(); -// System.out.println(response); -// } -// -//} diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go index 48d309fb9d..e923b62c63 100644 --- a/flow/connectors/iceberg/iceberg.go +++ b/flow/connectors/iceberg/iceberg.go @@ -277,8 +277,7 @@ func (c *IcebergConnector) ValidateCheck(ctx context.Context) error { if err != nil { return err } - c.logger.Debug("Created iceberg table", slog.String("table", table.TableName)) - c.logger.Debug("Will try to drop iceberg table", "table", tableName) + c.logger.Debug("Created iceberg table, will try to drop it now", "table", tableName) dropTable, err := c.proxyClient.DropTable(ctx, &protos.DropTableRequest{ TableInfo: &protos.TableInfo{ From 5a621685bd571885fed47ca0a2743a4fea8774ea Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Wed, 12 Jun 2024 01:27:53 +0530 Subject: [PATCH 07/31] fix: build and add multi-stage docker file --- .dockerignore | 9 ++ flow-jvm/.dockerignore | 18 ++- flow-jvm/README.md | 7 +- flow-jvm/src/main/docker/Dockerfile | 124 ++++++++++++++++++ .../catalog/mapper/CatalogConfigMapper.java | 2 +- flow-jvm/src/main/resources/application.yaml | 3 - flow/connectors/iceberg/iceberg.go | 2 +- 7 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 flow-jvm/src/main/docker/Dockerfile diff --git a/.dockerignore b/.dockerignore index c182d1d56d..f88740ad41 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,12 @@ ui/.next .gitignore .github .gitmodules + + +build +*.jar +.idea +.gradle +.dockerignore +.sdkmanrc + diff --git a/flow-jvm/.dockerignore b/flow-jvm/.dockerignore index 4361d2fb38..94b4e16661 100644 --- a/flow-jvm/.dockerignore +++ b/flow-jvm/.dockerignore @@ -1,5 +1,13 @@ -* -!build/*-runner -!build/*-runner.jar -!build/lib/* -!build/quarkus-app/* \ No newline at end of file +#* +#!build/*-runner +#!build/*-runner.jar +#!build/lib/* +#!build/quarkus-app/* +build +*.jar +.idea +.gradle +src/main/docker +.dockerignore +.git +.sdkmanrc diff --git a/flow-jvm/README.md b/flow-jvm/README.md index 73983d501a..c114a50433 100644 --- a/flow-jvm/README.md +++ b/flow-jvm/README.md @@ -10,7 +10,12 @@ This project uses Quarkus, the Supersonic Subatomic Java Framework. If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ . -## Running the application in dev mode + +## Generate the java code from proto files +```shell script +./gradlew quarkusGenerateCode +``` +## Running the application in dev mode (`quarkusGenerateCode` can be skipped) You can run your application in dev mode that enables live coding using: ```shell script diff --git a/flow-jvm/src/main/docker/Dockerfile b/flow-jvm/src/main/docker/Dockerfile new file mode 100644 index 0000000000..bf418f42f5 --- /dev/null +++ b/flow-jvm/src/main/docker/Dockerfile @@ -0,0 +1,124 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode +# +# Before building the container image run: +# +# ./gradlew build +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/flow-jvm-jvm . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/flow-jvm-jvm +# +# If you want to include the debug port into your docker image +# you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. +# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 +# when running the container +# +# Then run the container using : +# +# docker run -i --rm -p 8080:8080 quarkus/flow-jvm-jvm +# +# This image uses the `run-java.sh` script to run the application. +# This scripts computes the command line to execute your Java application, and +# includes memory/GC tuning. +# You can configure the behavior using the following environment properties: +# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class") +# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options +# in JAVA_OPTS (example: "-Dsome.property=foo") +# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is +# used to calculate a default maximal heap memory based on a containers restriction. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio +# of the container available memory as set here. The default is `50` which means 50% +# of the available memory is used as an upper boundary. You can skip this mechanism by +# setting this value to `0` in which case no `-Xmx` option is added. +# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This +# is used to calculate a default initial heap memory based on the maximum heap memory. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio +# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx` +# is used as the initial heap size. You can skip this mechanism by setting this value +# to `0` in which case no `-Xms` option is added (example: "25") +# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS. +# This is used to calculate the maximum value of the initial heap memory. If used in +# a container without any memory constraints for the container then this option has +# no effect. If there is a memory constraint then `-Xms` is limited to the value set +# here. The default is 4096MB which means the calculated value of `-Xms` never will +# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096") +# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output +# when things are happening. This option, if set to true, will set +# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true"). +# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example: +# true"). +# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787"). +# - CONTAINER_CORE_LIMIT: A calculated core limit as described in +# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2") +# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024"). +# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion. +# (example: "20") +# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking. +# (example: "40") +# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection. +# (example: "4") +# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus +# previous GC times. (example: "90") +# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20") +# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100") +# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should +# contain the necessary JRE command-line options to specify the required GC, which +# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC). +# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080") +# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080") +# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be +# accessed directly. (example: "foo.example.com,bar.example.com") +# +### +FROM gradle:8.8.0-jdk21 as builder + + + +WORKDIR /home/gradle/work + +# Copy the project files +COPY --chown=gradle:gradle flow-jvm/gradle gradle +COPY --chown=gradle:gradle flow-jvm/*.gradle flow-jvm/gradle.properties ./ + +# Gathers all proto files +COPY --chown=gradle:gradle protos/*.proto ../protos/ +# Adds any extra configuration files if needed for build +COPY --chown=gradle:gradle flow-jvm/src/main/resources ./src/main/resources + +# Only generate the code from the proto files +RUN gradle quarkusGenerateCode -x quarkusGenerateCodeDev --stacktrace --info + + +COPY --chown=gradle:gradle flow-jvm . +# Finally build the project +RUN gradle build -x quarkusGenerateCode -x quarkusGenerateCodeDev --stacktrace --info + + +FROM registry.access.redhat.com/ubi8/openjdk-21:1.19 as runner + + + +ENV LANGUAGE='en_US:en' + + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --from=builder --chown=185 /home/gradle/work/build/quarkus-app/lib/ /deployments/lib/ +COPY --from=builder --chown=185 /home/gradle/work/build/quarkus-app/*.jar /deployments/ +COPY --from=builder --chown=185 /home/gradle/work/build/quarkus-app/app/ /deployments/app/ +COPY --from=builder --chown=185 /home/gradle/work/build/quarkus-app/quarkus/ /deployments/quarkus/ + +# GRPC Port can be changed via FLOW_JVM_PORT env var +EXPOSE 9801 +USER 185 +ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + +ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ] + diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java index a28b399334..52adaaa9b3 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/CatalogConfigMapper.java @@ -21,7 +21,7 @@ protected Map mapCommon(CommonIcebergCatalog config) { if (config.hasCacheEnabled()) { builder.put(CatalogProperties.CACHE_ENABLED, String.valueOf(config.getCacheEnabled())); } - builder.putAll(config.getAdditionalPropertiesMap()); +// builder.putAll(config.getAdditionalPropertiesMap()); return builder.build(); } diff --git a/flow-jvm/src/main/resources/application.yaml b/flow-jvm/src/main/resources/application.yaml index 41d67118b8..82f75fd4fb 100644 --- a/flow-jvm/src/main/resources/application.yaml +++ b/flow-jvm/src/main/resources/application.yaml @@ -21,7 +21,4 @@ quarkus: http: port: ${FLOW_JVM_PORT:9801} - test: - continuous-testing: - enabled: disabled diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go index e923b62c63..9e8988e877 100644 --- a/flow/connectors/iceberg/iceberg.go +++ b/flow/connectors/iceberg/iceberg.go @@ -242,7 +242,7 @@ func (c *IcebergConnector) ValidateCheck(ctx context.Context) error { // Create a table with a random name based on current time tableName := fmt.Sprintf("__peerdb_test_flow_%d", time.Now().Unix()) c.logger.Debug("Will try to create iceberg table", "table", tableName) - table, err := c.proxyClient.CreateTable(ctx, + _, err := c.proxyClient.CreateTable(ctx, &protos.CreateTableRequest{ TableInfo: &protos.TableInfo{ Namespace: nil, From c28c10e7989b10f2619bafac61f988c3db8ef502 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 13 Jun 2024 05:59:06 +0530 Subject: [PATCH 08/31] feat(iceberg): add hive3 support and test GCS integration - additionally add ci build actions --- .github/workflows/flow-jvm-build.yml | 33 +++++++++++++++++++ docker-bake.hcl | 17 +++++++++- flow-jvm/build.gradle | 23 +++++++++++-- .../jvm/iceberg/catalog/CatalogLoader.java | 13 ++++++-- .../catalog/io/mapper/FixedS3FileIO.java | 10 ++++++ .../catalog/io/mapper/GCSIOConfigMapper.java | 12 ++----- .../catalog/io/mapper/S3IOConfigMapper.java | 4 +++ .../catalog/mapper/HiveConfigMapper.java | 13 ++++++-- .../jvm/iceberg/service/IcebergService.java | 17 +++++++++- flow-jvm/src/main/resources/application.yaml | 18 ++++++++-- flow/connectors/iceberg/iceberg.go | 4 ++- protos/peers.proto | 14 ++++---- .../Dockerfile => stacks/jvm.Dockerfile | 18 +++------- 13 files changed, 154 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/flow-jvm-build.yml create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FixedS3FileIO.java rename flow-jvm/src/main/docker/Dockerfile => stacks/jvm.Dockerfile (95%) diff --git a/.github/workflows/flow-jvm-build.yml b/.github/workflows/flow-jvm-build.yml new file mode 100644 index 0000000000..c1e661721f --- /dev/null +++ b/.github/workflows/flow-jvm-build.yml @@ -0,0 +1,33 @@ +name: Build & Test Flow JVM + +on: + push: + branches: [main] + pull_request: + branches: [main] + paths: [flow-jvm/**, protos/**, .sdkmanrc] + +jobs: + build-test: + name: Build & Test Flow JVM + strategy: + matrix: + runner: [ubicloud-standard-2-ubuntu-2204-arm] + runs-on: ${{ matrix.runner }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Setup SDKMAN and Install Dependencies + uses: sdkman/sdkman-action@main + id: sdkman + - name: Check dependency versions + run: | + sdk version + java -version + gradle -version + quarkus --version + working-directory: flow-jvm + - name: Build + working-directory: flow-jvm + run: ./gradlew build test --stacktrace --info diff --git a/docker-bake.hcl b/docker-bake.hcl index 6e6098ca14..538e652c83 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -16,7 +16,8 @@ group "default" { "flow-worker", "flow-api", "flow-snapshot-worker", - "peerdb-ui" + "peerdb-ui", + "flow-jvm", ] } @@ -90,3 +91,17 @@ target "peerdb-ui" { "${REGISTRY}/peerdb-ui:${SHA_SHORT}", ] } + +target "flow-jvm" { + context = "." + dockerfile = "stacks/jvm.Dockerfile" + target = "runner" + platforms = [ + "linux/amd64", + "linux/arm64", + ] + tags = [ + "${REGISTRY}/flow-jvm:${TAG}", + "${REGISTRY}/flow-jvm:${SHA_SHORT}", + ] +} diff --git a/flow-jvm/build.gradle b/flow-jvm/build.gradle index 17f88341e1..d800144518 100644 --- a/flow-jvm/build.gradle +++ b/flow-jvm/build.gradle @@ -19,17 +19,22 @@ dependencies { implementation 'io.quarkus:quarkus-arc' implementation 'io.quarkus:quarkus-grpc' implementation 'io.quarkus:quarkus-rest' + implementation 'io.quarkus:quarkus-logging-json' implementation "org.apache.iceberg:iceberg-core:${icebergLibVersion}" implementation "org.apache.iceberg:iceberg-common:${icebergLibVersion}" implementation "org.apache.iceberg:iceberg-data:${icebergLibVersion}" implementation "org.apache.iceberg:iceberg-parquet:${icebergLibVersion}" + // This is forced due to version conflicts + implementation 'io.grpc:grpc-protobuf:1.63.0' + implementation 'org.apache.avro:avro:1.11.3' implementation 'org.apache.hadoop:hadoop-client:3.4.0' + implementation 'org.apache.hadoop:hadoop-common:3.4.0' - // JDBC Drivers + // Drivers for JDBC Catalogs runtimeOnly 'org.postgresql:postgresql:42.7.3' // AWS Dependencies @@ -37,10 +42,23 @@ dependencies { runtimeOnly "org.apache.iceberg:iceberg-aws-bundle:${icebergLibVersion}" // runtimeOnly 'software.amazon.awssdk:bundle:2.25.60' +// // HIVE4 Dependencies +// implementation 'org.apache.hive:hive-iceberg-catalog:4.0.0' +// // DO NOT USE THE BELOW DEPENDENCIES https://github.com/apache/iceberg/issues/10429 +//// implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergLibVersion}" +//// runtimeOnly "org.apache.hive:hive-metastore:4.0.0" + + // HIVE 3 Dependencies + implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergLibVersion}" + runtimeOnly "org.apache.hive:hive-metastore:3.1.3" + + + + // GCP Dependencies implementation "org.apache.iceberg:iceberg-gcp:${icebergLibVersion}" // This is currently causing issues with GRPC versions mismatch -// runtimeOnly "org.apache.iceberg:iceberg-gcp-bundle:${icebergLibVersion}" + runtimeOnly "org.apache.iceberg:iceberg-gcp-bundle:${icebergLibVersion}" testImplementation 'io.quarkus:quarkus-junit5' @@ -69,7 +87,6 @@ compileTestJava { } - quarkus { quarkusBuildProperties.put("quarkus.grpc.codegen.proto-directory", "${project.projectDir}/../protos") } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java index d4b2d3c749..b9377fc31a 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/CatalogLoader.java @@ -9,10 +9,12 @@ import io.quarkus.logging.Log; import jakarta.inject.Inject; import jakarta.inject.Singleton; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.catalog.Catalog; import java.util.Collections; +import java.util.Map; @Singleton @@ -49,7 +51,14 @@ public Catalog loadCatalog(IcebergCatalog icebergCatalogConfig) { default -> throw new IllegalArgumentException("Unexpected value for catalog config: " + icebergCatalogConfig.getConfigCase()); }; - // TODO look at hadoop - return CatalogUtil.buildIcebergCatalog(icebergCatalogConfig.getCommonConfig().getName(), catalogConfig, null); + var hadoopConfiguration = getHadoopConfiguration(icebergCatalogConfig.getCommonConfig().getHadoopPropertiesMap()); + return CatalogUtil.buildIcebergCatalog(icebergCatalogConfig.getCommonConfig().getName(), catalogConfig, hadoopConfiguration); } + + private Configuration getHadoopConfiguration(Map hadoopConfig) { + var conf = new Configuration(); + hadoopConfig.forEach(conf::set); + return conf; + } + } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FixedS3FileIO.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FixedS3FileIO.java new file mode 100644 index 0000000000..4658a1cb6f --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/FixedS3FileIO.java @@ -0,0 +1,10 @@ +package io.peerdb.flow.jvm.iceberg.catalog.io.mapper; + +import org.apache.iceberg.aws.s3.S3FileIO; + +/** + * TODO This class should act as a delegate to the S3FileIO class, temporarily fixing till it is fixed upstream. + * Maybe a way to use it is to use Lombok's @Delegate functionality. + */ +public class FixedS3FileIO extends S3FileIO { +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/GCSIOConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/GCSIOConfigMapper.java index 3b9c836ddb..c84abba724 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/GCSIOConfigMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/GCSIOConfigMapper.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.peerdb.flow.peers.IcebergGCSIoConfig; import jakarta.enterprise.context.ApplicationScoped; +import org.apache.iceberg.gcp.GCPProperties; import java.util.Map; @@ -11,15 +12,8 @@ public class GCSIOConfigMapper extends FileIOConfigMapper { @Override protected Map mapSpecific(IcebergGCSIoConfig config) { // TODO complete this - var builder = ImmutableMap.builder(); -// .put(GCPProperties.GCS_PROJECT_ID, config.getAccessKeyId()) -// .put(S3FileIOProperties.SECRET_ACCESS_KEY, config.getSecretAccessKey()); -// if (config.hasEndpoint()) { -// builder.put(S3FileIOProperties.ENDPOINT, config.getEndpoint()); -// } -// if (config.hasPathStyleAccess()) { -// builder.put(S3FileIOProperties.PATH_STYLE_ACCESS, config.getPathStyleAccess()); -// } + var builder = ImmutableMap.builder() + .put(GCPProperties.GCS_PROJECT_ID, config.getProjectId()); return builder.build(); } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java index 986e267902..2f3956cbd1 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java @@ -20,6 +20,10 @@ protected Map mapSpecific(IcebergS3IoConfig config) { if (config.hasPathStyleAccess()) { builder.put(S3FileIOProperties.PATH_STYLE_ACCESS, config.getPathStyleAccess()); } + if (config.hasCrossRegionAccessEnabled()) { + // This only works when https://github.com/apache/iceberg/issues/9785 is fixed or {@link io.peerdb.flow.jvm.iceberg.catalog.io.mapper.FixedS3FileIO} is added + builder.put("s3.cross-region-access-enabled", config.getCrossRegionAccessEnabled()); + } return builder.build(); } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java index ba37344df3..67f9a431b3 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/mapper/HiveConfigMapper.java @@ -1,18 +1,25 @@ package io.peerdb.flow.jvm.iceberg.catalog.mapper; +import com.google.common.collect.ImmutableMap; import io.peerdb.flow.peers.HiveIcebergCatalog; import jakarta.inject.Singleton; import org.apache.iceberg.CatalogUtil; -import java.util.Collections; import java.util.Map; + +/** + * This requires the underlying thrift connection like thrift://localhost:9083 + */ + @Singleton public class HiveConfigMapper extends CatalogConfigMapper { @Override protected Map mapSpecific(HiveIcebergCatalog config) { - // TODO complete this - return Collections.emptyMap(); + return ImmutableMap.builder() + // TODO add these if needed +// .put(HiveCatalog.HMS_DB_OWNER, "hive") + .build(); } @Override diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index 56368428af..581a429f5e 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -16,9 +16,11 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.avro.AvroSchemaUtil; +import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.IcebergGenerics; import org.apache.iceberg.data.Record; +import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.io.TaskWriter; import org.apache.iceberg.io.WriteResult; @@ -81,8 +83,21 @@ public Table createTable(TableInfo tableInfo, String schema) { // var icebergSchema = new Schema(fieldList, primaryKeyFieldIds); var icebergSchema = typeSchema; Preconditions.checkArgument(icebergSchema.asStruct().equals(typeSchema.asStruct()), "Primary key based schema not equivalent to type schema [%s!=%s]", icebergSchema.asStruct(), typeSchema.asStruct()); + + var tableIdentifier = getTableIdentifier(tableInfo); + // We create the namespace if needed + if (!tableIdentifier.namespace().isEmpty() && catalog instanceof SupportsNamespaces namespacedCatalog && namespacedCatalog.namespaceExists(tableIdentifier.namespace())) { + try { + Log.infof("Creating namespace %s", tableIdentifier.namespace()); + namespacedCatalog.createNamespace(tableIdentifier.namespace()); + } catch (AlreadyExistsException e) { + Log.warnf("Namespace %s already exists, skipping", tableIdentifier.namespace()); + } catch (UnsupportedOperationException e) { + Log.warnf("Namespace creation not supported by catalog %s, skipping", icebergCatalog); + } + } Log.infof("Will now create table %s", tableInfo.getTableName()); - var table = catalog.createTable(getTableIdentifier(tableInfo), icebergSchema); + var table = catalog.createTable(tableIdentifier, icebergSchema); Log.infof("Created table %s", tableInfo.getTableName()); return table; } diff --git a/flow-jvm/src/main/resources/application.yaml b/flow-jvm/src/main/resources/application.yaml index 82f75fd4fb..089701d2a7 100644 --- a/flow-jvm/src/main/resources/application.yaml +++ b/flow-jvm/src/main/resources/application.yaml @@ -15,10 +15,24 @@ quarkus: category: "io.peerdb": level: ${PEERDB_LOG_LEVEL:INFO} - min-level: ${PEERDB_LOG_LEVEL:TRACE} + min-level: ${PEERDB_LOG_LEVEL:INFO} + "io.peerdb.flow.jvm.iceberg.catalog": + level: ${PEERDB_ICEBERG_CATALOG_LOG_LEVEL:INFO} + min-level: ${PEERDB_ICEBERG_CATALOG_LOG_LEVEL:INFO} + console: - level: ${PEERDB_LOG_LEVEL:TRACE} + level: ${PEERDB_LOG_LEVEL:INFO} http: port: ${FLOW_JVM_PORT:9801} +"%dev": + quarkus: + log: + console: + json: false +"%test": + quarkus: + log: + console: + json: false diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go index 9e8988e877..34a397212f 100644 --- a/flow/connectors/iceberg/iceberg.go +++ b/flow/connectors/iceberg/iceberg.go @@ -240,7 +240,9 @@ func (c *IcebergConnector) Close() error { func (c *IcebergConnector) ValidateCheck(ctx context.Context) error { // Create a table with a random name based on current time - tableName := fmt.Sprintf("__peerdb_test_flow_%d", time.Now().Unix()) + // TODO ask for namespace in the peer settings and use that instead of __peerdb_test + // Can also ask for a boolean if provided namespace is to be created or not + tableName := fmt.Sprintf("__peerdb_test_%d.__peerdb_test_flow_%d", time.Now().Unix(), time.Now().Unix()) c.logger.Debug("Will try to create iceberg table", "table", tableName) _, err := c.proxyClient.CreateTable(ctx, &protos.CreateTableRequest{ diff --git a/protos/peers.proto b/protos/peers.proto index efc5d4309e..956b19c590 100644 --- a/protos/peers.proto +++ b/protos/peers.proto @@ -5,7 +5,6 @@ package peerdb_peers; option java_multiple_files = true; option java_package = "io.peerdb.flow.peers"; - message SSHConfig { string host = 1; uint32 port = 2; @@ -108,7 +107,7 @@ message S3Config { optional string endpoint = 6; } -message ClickhouseConfig{ +message ClickhouseConfig { string host = 1; uint32 port = 2; string user = 3; @@ -173,10 +172,11 @@ message IcebergConfig { message IcebergS3IoConfig { optional string access_key_id = 1; optional string secret_access_key = 2; - optional string region = 3; - optional string endpoint = 4; + optional string endpoint = 3; // Set to true to use for services like MinIO - optional string path_style_access = 5; + optional string path_style_access = 4; + // For enabling cross region bucket access, works when https://github.com/apache/iceberg/issues/9785 is resolved + optional string cross_region_access_enabled = 5; } message IcebergGCSIoConfig { @@ -198,8 +198,9 @@ message CommonIcebergCatalog { string warehouse_location = 3; optional int32 client_pool_size = 4; optional bool cache_enabled = 5; + map hadoop_properties = 6; // This helps in testing, where we can pass additional properties to the catalog -// map additional_properties = 6; + // map additional_properties = 6; } message HiveIcebergCatalog {} @@ -232,7 +233,6 @@ message IcebergCatalog { } } - enum DBType { BIGQUERY = 0; SNOWFLAKE = 1; diff --git a/flow-jvm/src/main/docker/Dockerfile b/stacks/jvm.Dockerfile similarity index 95% rename from flow-jvm/src/main/docker/Dockerfile rename to stacks/jvm.Dockerfile index bf418f42f5..d7b5a90576 100644 --- a/flow-jvm/src/main/docker/Dockerfile +++ b/stacks/jvm.Dockerfile @@ -1,18 +1,6 @@ #### # This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode # -# Before building the container image run: -# -# ./gradlew build -# -# Then, build the image with: -# -# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/flow-jvm-jvm . -# -# Then run the container using: -# -# docker run -i --rm -p 8080:8080 quarkus/flow-jvm-jvm -# # If you want to include the debug port into your docker image # you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. # Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 @@ -87,6 +75,9 @@ WORKDIR /home/gradle/work COPY --chown=gradle:gradle flow-jvm/gradle gradle COPY --chown=gradle:gradle flow-jvm/*.gradle flow-jvm/gradle.properties ./ +# Build once to cache dependencies +RUN gradle build -x quarkusGenerateCode -x quarkusGenerateCodeDev --stacktrace --info + # Gathers all proto files COPY --chown=gradle:gradle protos/*.proto ../protos/ # Adds any extra configuration files if needed for build @@ -98,7 +89,7 @@ RUN gradle quarkusGenerateCode -x quarkusGenerateCodeDev --stacktrace --info COPY --chown=gradle:gradle flow-jvm . # Finally build the project -RUN gradle build -x quarkusGenerateCode -x quarkusGenerateCodeDev --stacktrace --info +RUN gradle build --stacktrace --info FROM registry.access.redhat.com/ubi8/openjdk-21:1.19 as runner @@ -119,6 +110,7 @@ EXPOSE 9801 USER 185 ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" +ENV PEERDB_LOG_LEVEL="INFO" ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ] From f72adde2796881da8b63ffa2fa9d84341750e48d Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 13 Jun 2024 06:01:06 +0530 Subject: [PATCH 09/31] chore: remove check for sdkman version --- .github/workflows/flow-jvm-build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flow-jvm-build.yml b/.github/workflows/flow-jvm-build.yml index c1e661721f..f205d1a953 100644 --- a/.github/workflows/flow-jvm-build.yml +++ b/.github/workflows/flow-jvm-build.yml @@ -23,11 +23,16 @@ jobs: id: sdkman - name: Check dependency versions run: | - sdk version + export JAVA_HOME=$HOME/.sdkman/candidates/java/current + echo "JAVA_HOME=${JAVA_HOME}" >> $GITHUB_ENV java -version gradle -version quarkus --version working-directory: flow-jvm + - name: Download/Setup gradle wrapper + working-directory: flow-jvm + run: | + gradle wrapper - name: Build working-directory: flow-jvm run: ./gradlew build test --stacktrace --info From 77d18f3c9bfb09ec4e72328c909a9cd05e172204 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 13 Jun 2024 06:30:16 +0530 Subject: [PATCH 10/31] chore: switch from sdkman to setupo-java --- .github/workflows/flow-jvm-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/flow-jvm-build.yml b/.github/workflows/flow-jvm-build.yml index f205d1a953..6317e4a9d3 100644 --- a/.github/workflows/flow-jvm-build.yml +++ b/.github/workflows/flow-jvm-build.yml @@ -18,16 +18,16 @@ jobs: - name: checkout uses: actions/checkout@v4 - - name: Setup SDKMAN and Install Dependencies - uses: sdkman/sdkman-action@main - id: sdkman + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + cache: 'gradle' + - name: Check dependency versions run: | - export JAVA_HOME=$HOME/.sdkman/candidates/java/current - echo "JAVA_HOME=${JAVA_HOME}" >> $GITHUB_ENV java -version gradle -version - quarkus --version working-directory: flow-jvm - name: Download/Setup gradle wrapper working-directory: flow-jvm From 9db73e71ccca8181cbf1c8ab24d288fc374b1068 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 13 Jun 2024 07:33:20 +0530 Subject: [PATCH 11/31] chore: add explicit proto dependency --- .github/workflows/flow-jvm-build.yml | 2 +- flow-jvm/build.gradle | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flow-jvm-build.yml b/.github/workflows/flow-jvm-build.yml index 6317e4a9d3..4e8372ee74 100644 --- a/.github/workflows/flow-jvm-build.yml +++ b/.github/workflows/flow-jvm-build.yml @@ -35,4 +35,4 @@ jobs: gradle wrapper - name: Build working-directory: flow-jvm - run: ./gradlew build test --stacktrace --info + run: ./gradlew quarkusGenerateCode --stacktrace --info && ./gradlew build test --stacktrace --info diff --git a/flow-jvm/build.gradle b/flow-jvm/build.gradle index d800144518..f6d344c105 100644 --- a/flow-jvm/build.gradle +++ b/flow-jvm/build.gradle @@ -28,6 +28,7 @@ dependencies { // This is forced due to version conflicts implementation 'io.grpc:grpc-protobuf:1.63.0' + implementation 'com.google.protobuf:protobuf-java:4.27.1' implementation 'org.apache.avro:avro:1.11.3' @@ -89,4 +90,5 @@ compileTestJava { quarkus { quarkusBuildProperties.put("quarkus.grpc.codegen.proto-directory", "${project.projectDir}/../protos") + quarkusBuildProperties.put("quarkus.grpc.codegen.exclude-filter", ".*") } From 1bbe868395f2ff4f38320d419826bf9671e82ef0 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:03:43 +0530 Subject: [PATCH 12/31] chore: remove extra log setting --- flow-jvm/src/main/resources/application.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/flow-jvm/src/main/resources/application.yaml b/flow-jvm/src/main/resources/application.yaml index 089701d2a7..98f2d15614 100644 --- a/flow-jvm/src/main/resources/application.yaml +++ b/flow-jvm/src/main/resources/application.yaml @@ -16,9 +16,6 @@ quarkus: "io.peerdb": level: ${PEERDB_LOG_LEVEL:INFO} min-level: ${PEERDB_LOG_LEVEL:INFO} - "io.peerdb.flow.jvm.iceberg.catalog": - level: ${PEERDB_ICEBERG_CATALOG_LOG_LEVEL:INFO} - min-level: ${PEERDB_ICEBERG_CATALOG_LOG_LEVEL:INFO} console: level: ${PEERDB_LOG_LEVEL:INFO} From 7fe52e71e52e1a553055a785b6b432f895a1d981 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:29:14 +0530 Subject: [PATCH 13/31] feat: Enhance logging and dependency configurations - Add logging adapters to prevent duplicate slf4j binding warnings - Exclude specific modules to avoid conflicts --- flow-jvm/build.gradle | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/flow-jvm/build.gradle b/flow-jvm/build.gradle index f6d344c105..a741a4a9c7 100644 --- a/flow-jvm/build.gradle +++ b/flow-jvm/build.gradle @@ -21,6 +21,12 @@ dependencies { implementation 'io.quarkus:quarkus-rest' implementation 'io.quarkus:quarkus-logging-json' + // Logging adapter for dependencies, also prevents duplicate slf4j binding warnings https://quarkus.io/guides/logging#logging-apis + implementation("org.jboss.logging:commons-logging-jboss-logging") + implementation("org.jboss.logmanager:log4j-jboss-logmanager") + implementation("org.jboss.logmanager:log4j2-jboss-logmanager") + implementation("org.jboss.slf4j:slf4j-jboss-logmanager") + implementation "org.apache.iceberg:iceberg-core:${icebergLibVersion}" implementation "org.apache.iceberg:iceberg-common:${icebergLibVersion}" implementation "org.apache.iceberg:iceberg-data:${icebergLibVersion}" @@ -67,6 +73,19 @@ dependencies { } + +configurations.configureEach { +// // ch.qos.logback:logback-core +// exclude group: 'ch.qos.logback', module: 'logback-core' + // ch.qos.logback:logback-classic + exclude group: 'ch.qos.logback', module: 'logback-classic' + // org.apache.logging.log4j:log4j-slf4j-impl + exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl' + // org.slf4j:slf4j-reload4j + exclude group: 'org.slf4j', module: 'slf4j-reload4j' +} + + group 'io.peerdb' version '0.0.1-SNAPSHOT' From de5c69c39ce8a7fe1ea4d03ddcea61767dd86a47 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:36:28 +0530 Subject: [PATCH 14/31] fix: Correct namespace existence check in IcebergService - Modify the condition in IcebergService to correctly check if a namespace exists before creating it. --- .../java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index 581a429f5e..9f7ceb9901 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -86,7 +86,7 @@ public Table createTable(TableInfo tableInfo, String schema) { var tableIdentifier = getTableIdentifier(tableInfo); // We create the namespace if needed - if (!tableIdentifier.namespace().isEmpty() && catalog instanceof SupportsNamespaces namespacedCatalog && namespacedCatalog.namespaceExists(tableIdentifier.namespace())) { + if (!tableIdentifier.namespace().isEmpty() && catalog instanceof SupportsNamespaces namespacedCatalog && !namespacedCatalog.namespaceExists(tableIdentifier.namespace())) { try { Log.infof("Creating namespace %s", tableIdentifier.namespace()); namespacedCatalog.createNamespace(tableIdentifier.namespace()); From efef82657070b022520ae0fba340d99ce07872ba Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:01:11 +0530 Subject: [PATCH 15/31] feat(iceberg): add streaming support to improve performance and memory footprint (#1835) This uses streams instead of sending all the records in 1 go. --- .github/actions/genprotos/action.yml | 2 +- flow-jvm/README.md | 6 + .../jvm/DefaultExceptionHandlerProvider.java | 2 +- .../jvm/iceberg/resource/IcebergResource.java | 11 ++ .../jvm/iceberg/service/IcebergService.java | 38 ++++- flow-jvm/src/main/resources/application.yaml | 3 +- flow/connectors/iceberg/qrep.go | 130 ++++++++++++++++-- protos/flow-jvm.proto | 26 +++- 8 files changed, 194 insertions(+), 24 deletions(-) diff --git a/.github/actions/genprotos/action.yml b/.github/actions/genprotos/action.yml index abedf058b6..142856dc37 100644 --- a/.github/actions/genprotos/action.yml +++ b/.github/actions/genprotos/action.yml @@ -12,7 +12,7 @@ runs: ./flow/generated/protos ./nexus/pt/src/gen ./ui/grpc_generated - key: ${{ runner.os }}-build-genprotos-${{ hashFiles('./protos/peers.proto', './protos/flow.proto', './protos/route.proto') }} + key: ${{ runner.os }}-build-genprotos-${{ hashFiles('./protos/peers.proto', './protos/flow.proto', './protos/route.proto', './protos/flow-jvm.proto') }} - if: steps.cache.outputs.cache-hit != 'true' uses: actions/setup-go@v5 diff --git a/flow-jvm/README.md b/flow-jvm/README.md index c114a50433..44b67d48df 100644 --- a/flow-jvm/README.md +++ b/flow-jvm/README.md @@ -15,6 +15,12 @@ If you want to learn more about Quarkus, please visit its website: https://quark ```shell script ./gradlew quarkusGenerateCode ``` + +or +```shell +./gradlew clean quarkusGenerateCode +``` + ## Running the application in dev mode (`quarkusGenerateCode` can be skipped) You can run your application in dev mode that enables live coding using: diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java index 98894fa438..e41b2103a7 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java @@ -26,7 +26,7 @@ public ExceptionHandler createHandler(ServerCall.List @Override public Throwable transform(Throwable t) { invoked = true; - Log.errorf(t, "Received error in gRPC call"); + Log.errorf(t, "Received error in gRPC call %s", t.getMessage()); return toStatusException(t); } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java index 30b8cc11f1..aea32f57f4 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java @@ -5,7 +5,9 @@ import io.quarkus.grpc.GrpcService; import io.smallrye.common.annotation.Blocking; import io.smallrye.common.annotation.RunOnVirtualThread; +import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.Uni; +import io.smallrye.mutiny.unchecked.Unchecked; import jakarta.inject.Inject; import java.util.Optional; @@ -67,4 +69,13 @@ public Uni appendRecords(AppendRecordsRequest request) { icebergService.processAppendRecordsRequest(request)) .build()); } + + @Blocking + @Override + public Uni streamingAppendRecords(Multi request) { + return Uni.createFrom().item(Unchecked.supplier(() -> AppendRecordsStreamResponse.newBuilder() + .setSuccess(icebergService.processAppendRecordsStreamRequest(request)) + .build())); + + } } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index 9f7ceb9901..cf1d63982e 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -9,6 +9,9 @@ import io.peerdb.flow.jvm.iceberg.lock.LockManager; import io.peerdb.flow.jvm.iceberg.writer.RecordWriterFactory; import io.quarkus.logging.Log; +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; +import io.smallrye.mutiny.unchecked.Unchecked; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.apache.iceberg.ContentFile; @@ -45,6 +48,7 @@ public class IcebergService { RecordWriterFactory recordWriterFactory; private static void writeRecordStream(Stream recordStream, AvroIcebergRecordConverter converter, TaskWriter writer) { + // use non parallel and use multi everywhere recordStream.parallel().map(insertRecord -> { try { return converter.toIcebergRecord(insertRecord.getRecord().toByteArray()); @@ -109,20 +113,41 @@ public boolean dropTable(TableInfo tableInfo, boolean purge) { } public boolean processAppendRecordsRequest(AppendRecordsRequest request) { - return appendRecords(request.getTableInfo(), - request.getSchema(), + return appendRecords(request.getTableHeader().getTableInfo(), + request.getTableHeader().getSchema(), request.getRecordsList(), - Optional.ofNullable(request.hasIdempotencyKey() ? request.getIdempotencyKey() : null)); + Optional.ofNullable(request.getTableHeader().hasIdempotencyKey() ? request.getTableHeader().getIdempotencyKey() : null)); + } + + public boolean processAppendRecordsStreamRequest(Multi request) { + var firstMessage = Uni.createFrom().multi(request).await().indefinitely(); + if (!firstMessage.hasTableHeader()) { + throw new IllegalArgumentException("TableHeader should be present in the first message"); + } + var tableHeader = firstMessage.getTableHeader(); + var tableInfo = tableHeader.getTableInfo(); + var avroSchema = tableHeader.getSchema(); + var insertRecordStream = request.map(Unchecked.function(message -> { + if (message.hasRecord()) { + return message.getRecord(); + } else { + throw new IllegalArgumentException("Only InsertRecord is supported"); + } + })).subscribe().asStream(); + return appendRecords(tableInfo, avroSchema, insertRecordStream, Optional.ofNullable(tableHeader.hasIdempotencyKey() ? tableHeader.getIdempotencyKey() : null)); } private boolean appendRecords(TableInfo tableInfo, String avroSchema, List insertRecords, Optional idempotencyKey) { + return appendRecords(tableInfo, avroSchema, insertRecords.stream(), idempotencyKey); + } + + public boolean appendRecords(TableInfo tableInfo, String avroSchema, Stream recordStream, Optional idempotencyKey) { var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); var table = icebergCatalog.loadTable(getTableIdentifier(tableInfo)); if (isAppendAlreadyDone(table, idempotencyKey)) { return true; } - var recordStream = insertRecords.stream(); Log.infof("Converting append records to data files for table %s", table.name()); var dataFiles = getAppendDataFiles(avroSchema, table, recordStream); @@ -165,7 +190,7 @@ private boolean appendRecords(TableInfo tableInfo, String avroSchema, List")); } } @@ -187,8 +212,7 @@ private DataFile[] getAppendDataFiles(String avroSchema, Table table, Stream idempotencyKey) { diff --git a/flow-jvm/src/main/resources/application.yaml b/flow-jvm/src/main/resources/application.yaml index 98f2d15614..c73efe1bb5 100644 --- a/flow-jvm/src/main/resources/application.yaml +++ b/flow-jvm/src/main/resources/application.yaml @@ -15,7 +15,8 @@ quarkus: category: "io.peerdb": level: ${PEERDB_LOG_LEVEL:INFO} - min-level: ${PEERDB_LOG_LEVEL:INFO} + # This is build time + min-level: ${PEERDB_MIN_LOG_LEVEL:DEBUG} console: level: ${PEERDB_LOG_LEVEL:INFO} diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go index 4b8ca14bc2..3cbcf1dfa7 100644 --- a/flow/connectors/iceberg/qrep.go +++ b/flow/connectors/iceberg/qrep.go @@ -19,6 +19,16 @@ func (c *IcebergConnector) SyncQRepRecords( config *protos.QRepConfig, partition *protos.QRepPartition, stream *model.QRecordStream, +) (int, error) { + return c.streamRecords(ctx, config, partition, stream) +} + +//nolint:unused +func (c *IcebergConnector) sendRecordsJoined( + ctx context.Context, + config *protos.QRepConfig, + partition *protos.QRepPartition, + stream *model.QRecordStream, ) (int, error) { schema := stream.Schema() @@ -69,16 +79,17 @@ func (c *IcebergConnector) SyncQRepRecords( appendRecordsResponse, err := c.proxyClient.AppendRecords(ctx, &protos.AppendRecordsRequest{ - TableInfo: &protos.TableInfo{ - // Namespace: nil, - TableName: dstTableName, - IcebergCatalog: c.config.CatalogConfig, - // PrimaryKey: nil, - + TableHeader: &protos.AppendRecordTableHeader{ + TableInfo: &protos.TableInfo{ + // Namespace: nil, + TableName: dstTableName, + IcebergCatalog: c.config.CatalogConfig, + // PrimaryKey: nil, + }, + Schema: avroSchema.Schema, + IdempotencyKey: &requestIdempotencyKey, }, - Schema: avroSchema.Schema, - Records: binaryRecords, - IdempotencyKey: &requestIdempotencyKey, + Records: binaryRecords, }, ) if err != nil { @@ -94,6 +105,107 @@ func (c *IcebergConnector) SyncQRepRecords( return len(binaryRecords), nil } +func (c *IcebergConnector) streamRecords( + ctx context.Context, + config *protos.QRepConfig, + partition *protos.QRepPartition, + stream *model.QRecordStream, +) (int, error) { + schema := stream.Schema() + + schema.Fields = addPeerMetaColumns(schema.Fields, config.SoftDeleteColName, config.SyncedAtColName) + dstTableName := config.DestinationTableIdentifier + + avroSchema, err := getAvroSchema(dstTableName, schema) + if err != nil { + return 0, err + } + + avroConverter := model.NewQRecordAvroConverter( + avroSchema, + protos.DBType_ICEBERG, + schema.GetColumnNames(), + logger.LoggerFromCtx(ctx), + ) + codec, err := goavro.NewCodec(avroSchema.Schema) + if err != nil { + return 0, fmt.Errorf("failed to create Avro codec: %w", err) + } + requestIdempotencyKey := fmt.Sprintf("_peerdb_qrep-%s-%s", config.FlowJobName, partition.PartitionId) + + tableHeader := &protos.AppendRecordTableHeader{ + TableInfo: &protos.TableInfo{ + // Namespace: nil, + TableName: dstTableName, + IcebergCatalog: c.config.CatalogConfig, + // PrimaryKey: nil, + }, + Schema: avroSchema.Schema, + IdempotencyKey: &requestIdempotencyKey, + } + recordStream, err := c.proxyClient.StreamingAppendRecords(ctx) + if err != nil { + return 0, err + } + + err = recordStream.Send(&protos.AppendRecordsStreamRequest{ + Command: &protos.AppendRecordsStreamRequest_TableHeader{ + TableHeader: tableHeader, + }, + }) + // TODO what to do with recordStream? + if err != nil { + return 0, err + } + + recordCount := 0 + for record := range stream.Records { + record = append(record, + // Add soft delete + qvalue.QValueBoolean{ + Val: false, + }, // add synced at colname + qvalue.QValueTimestampTZ{ + Val: time.Now(), + }) + + converted, err := avroConverter.Convert(record) + if err != nil { + return 0, err + } + binaryData := make([]byte, 0) + native, err := codec.BinaryFromNative(binaryData, converted) + if err != nil { + return 0, fmt.Errorf("failed to convert Avro map to binary: %w", err) + } + insertRecord := &protos.InsertRecord{ + Record: native, + } + err = recordStream.Send(&protos.AppendRecordsStreamRequest{ + Command: &protos.AppendRecordsStreamRequest_Record{ + Record: insertRecord, + }, + }) + if err != nil { + return 0, err + } + recordCount++ + } + + appendRecordsStreamResponse, err := recordStream.CloseAndRecv() + if err != nil { + return 0, err + } + + logger.LoggerFromCtx(ctx).Info("AppendRecordsResponse", slog.Any("response", appendRecordsStreamResponse.Success)) + + err = c.PostgresMetadata.FinishQRepPartition(ctx, partition, config.FlowJobName, time.Now()) + if err != nil { + return 0, err + } + return recordCount, nil +} + func getAvroSchema( dstTableName string, schema qvalue.QRecordSchema, diff --git a/protos/flow-jvm.proto b/protos/flow-jvm.proto index 6749f4134a..33ba2d9162 100644 --- a/protos/flow-jvm.proto +++ b/protos/flow-jvm.proto @@ -17,8 +17,9 @@ service IcebergProxyService { rpc InsertChanges(InsertChangesRequest) returns (InsertChangesResponse) {} rpc AppendRecords(AppendRecordsRequest) returns (AppendRecordsResponse) {} -} + rpc StreamingAppendRecords(stream AppendRecordsStreamRequest) returns (AppendRecordsStreamResponse) {} +} message TableInfo { repeated string namespace = 1; @@ -45,7 +46,6 @@ message DropTableResponse { bool success = 1; } - message CountRecordRequest { TableInfo table_info = 1; } @@ -95,13 +95,29 @@ message InsertChangesResponse { bool success = 1; } -message AppendRecordsRequest { +message AppendRecordTableHeader { TableInfo table_info = 1; string schema = 2; - repeated InsertRecord records = 3; - optional string idempotency_key = 4; + optional string idempotency_key = 3; +} + +message AppendRecordsRequest { + AppendRecordTableHeader table_header = 1; + repeated InsertRecord records = 2; } message AppendRecordsResponse { bool success = 1; } + + +message AppendRecordsStreamRequest { + oneof command { + AppendRecordTableHeader table_header = 1; + InsertRecord record = 2; + } +} + +message AppendRecordsStreamResponse { + bool success = 1; +} From d2ab4a3e87f7304d17223060feb93bc6487dae59 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:18:55 +0530 Subject: [PATCH 16/31] fix: path style access --- .../flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java | 2 +- protos/peers.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java index 2f3956cbd1..54a2c9045e 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/catalog/io/mapper/S3IOConfigMapper.java @@ -18,7 +18,7 @@ protected Map mapSpecific(IcebergS3IoConfig config) { builder.put(S3FileIOProperties.ENDPOINT, config.getEndpoint()); } if (config.hasPathStyleAccess()) { - builder.put(S3FileIOProperties.PATH_STYLE_ACCESS, config.getPathStyleAccess()); + builder.put(S3FileIOProperties.PATH_STYLE_ACCESS, String.valueOf(config.getPathStyleAccess())); } if (config.hasCrossRegionAccessEnabled()) { // This only works when https://github.com/apache/iceberg/issues/9785 is fixed or {@link io.peerdb.flow.jvm.iceberg.catalog.io.mapper.FixedS3FileIO} is added diff --git a/protos/peers.proto b/protos/peers.proto index 956b19c590..e5effee24e 100644 --- a/protos/peers.proto +++ b/protos/peers.proto @@ -174,7 +174,7 @@ message IcebergS3IoConfig { optional string secret_access_key = 2; optional string endpoint = 3; // Set to true to use for services like MinIO - optional string path_style_access = 4; + optional bool path_style_access = 4; // For enabling cross region bucket access, works when https://github.com/apache/iceberg/issues/9785 is resolved optional string cross_region_access_enabled = 5; } From 2e781de909d69b90764805ceb8c9dbef9473870c Mon Sep 17 00:00:00 2001 From: Amogh Bharadwaj Date: Tue, 18 Jun 2024 09:19:22 +0530 Subject: [PATCH 17/31] Iceberg peer UI (#1840) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ![Screenshot 2024-06-15 at 1 28 50 AM](https://github.com/PeerDB-io/peerdb/assets/65964360/630b0ac8-7cdc-413d-a436-f8340bad3a45) TODO: - [x] Add required asterisk - [ ] Make validate and create work - [x] Test and fix the wiring of all fields to Flow API - [x] Add form validation and error toast - [x] Add tips and helpful links --------- Co-authored-by: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> --- .../flow/jvm/RequestLoggingInterceptor.java | 2 +- .../jvm/iceberg/resource/IcebergResource.java | 6 +- .../jvm/iceberg/service/IcebergService.java | 91 ++++- .../jvm/iceberg/writer/AppendFilesWriter.java | 36 ++ flow-jvm/src/main/resources/application.yaml | 2 +- flow/connectors/iceberg/qrep.go | 12 +- flow/peerdbenv/features/iceberg.go | 20 ++ ui/app/api/peers/getTruePeer.ts | 8 +- ui/app/api/peers/route.ts | 7 + ui/app/dto/PeersDTO.ts | 4 +- ui/app/peers/create/[peerType]/handlers.ts | 7 +- ui/app/peers/create/[peerType]/helpers/ice.ts | 339 ++++++++++++++++++ ui/app/peers/create/[peerType]/page.tsx | 9 + ui/app/peers/create/[peerType]/schema.ts | 78 ++++ ui/components/PeerComponent.tsx | 3 + .../PeerForms/Iceberg/IcebergConfig.tsx | 267 ++++++++++++++ ui/components/SelectSource.tsx | 10 +- ui/public/images/iceberg.png | Bin 0 -> 17608 bytes 18 files changed, 876 insertions(+), 25 deletions(-) create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/AppendFilesWriter.java create mode 100644 flow/peerdbenv/features/iceberg.go create mode 100644 ui/app/peers/create/[peerType]/helpers/ice.ts create mode 100644 ui/components/PeerForms/Iceberg/IcebergConfig.tsx create mode 100644 ui/public/images/iceberg.png diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java index 28853557e1..473f05efe1 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java @@ -35,7 +35,7 @@ public CallListener(ServerCallHandler serverCallHandler, ServerCall @Override public void onMessage(ReqT message) { - Log.debugf("Received request for method: %s: {%s}", serverCall.getMethodDescriptor().getFullMethodName(), message); +// Log.debugf("Received request for method: %s: {%s}", serverCall.getMethodDescriptor().getFullMethodName(), message); super.onMessage(message); } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java index aea32f57f4..6e43eaafd0 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java @@ -7,7 +7,6 @@ import io.smallrye.common.annotation.RunOnVirtualThread; import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.Uni; -import io.smallrye.mutiny.unchecked.Unchecked; import jakarta.inject.Inject; import java.util.Optional; @@ -73,9 +72,6 @@ public Uni appendRecords(AppendRecordsRequest request) { @Blocking @Override public Uni streamingAppendRecords(Multi request) { - return Uni.createFrom().item(Unchecked.supplier(() -> AppendRecordsStreamResponse.newBuilder() - .setSuccess(icebergService.processAppendRecordsStreamRequest(request)) - .build())); - + return icebergService.appendRecordsAsync(request); } } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index cf1d63982e..8d1a5abc9a 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -7,6 +7,7 @@ import io.peerdb.flow.jvm.iceberg.avro.AvroIcebergRecordConverter; import io.peerdb.flow.jvm.iceberg.catalog.CatalogLoader; import io.peerdb.flow.jvm.iceberg.lock.LockManager; +import io.peerdb.flow.jvm.iceberg.writer.AppendFilesWriter; import io.peerdb.flow.jvm.iceberg.writer.RecordWriterFactory; import io.quarkus.logging.Log; import io.smallrye.mutiny.Multi; @@ -14,11 +15,13 @@ import io.smallrye.mutiny.unchecked.Unchecked; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import org.apache.commons.lang3.tuple.Pair; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.avro.AvroSchemaUtil; +import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.IcebergGenerics; @@ -34,6 +37,8 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @ApplicationScoped @@ -120,13 +125,17 @@ public boolean processAppendRecordsRequest(AppendRecordsRequest request) { } public boolean processAppendRecordsStreamRequest(Multi request) { - var firstMessage = Uni.createFrom().multi(request).await().indefinitely(); - if (!firstMessage.hasTableHeader()) { + Log.info("Received a Process Append Records Stream Request, will wait for first message"); + // Lets error out after 1 minute + var firstMessage = Uni.createFrom().multi(request).await().atMost(Duration.ofSeconds(30)); + if (firstMessage.getCommandCase() != AppendRecordsStreamRequest.CommandCase.TABLE_HEADER && !firstMessage.hasTableHeader()) { + Log.errorf("TableHeader should be present in the first message, found %s", firstMessage.getCommandCase()); throw new IllegalArgumentException("TableHeader should be present in the first message"); } var tableHeader = firstMessage.getTableHeader(); var tableInfo = tableHeader.getTableInfo(); var avroSchema = tableHeader.getSchema(); + Log.infof("Received a Process Append Records Stream Request for table %s, will publish stream for appending records", tableInfo.getTableName()); var insertRecordStream = request.map(Unchecked.function(message -> { if (message.hasRecord()) { return message.getRecord(); @@ -141,26 +150,72 @@ private boolean appendRecords(TableInfo tableInfo, String avroSchema, List recordStream, Optional idempotencyKey) { - var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); - var table = icebergCatalog.loadTable(getTableIdentifier(tableInfo)); + record AppendRecordTableContext(AppendRecordTableHeader tableHeader, Catalog catalog, Table table, + AppendFilesWriter appendFilesWriter) { + } - if (isAppendAlreadyDone(table, idempotencyKey)) { + class OperationDoneException extends RuntimeException { + } + + public Uni appendRecordsAsync(Multi request) { + // TODO the streaming code needs better error handling + var tableContext = new AtomicReference(); + var counter = new AtomicInteger(); + Multi appendRecordStream = request.map(record -> Pair.of(counter.getAndIncrement(), record)).map(Unchecked.function(pair -> { + var record = pair.getValue(); + if (record.getCommandCase() == AppendRecordsStreamRequest.CommandCase.TABLE_HEADER) { + var tableHeader = record.getTableHeader(); + var avroSchema = tableHeader.getSchema(); + var icebergCatalog = catalogLoader.loadCatalog(tableHeader.getTableInfo().getIcebergCatalog()); + var table = icebergCatalog.loadTable(getTableIdentifier(tableHeader.getTableInfo())); + if (isAppendAlreadyDone(table, Optional.ofNullable(tableHeader.hasIdempotencyKey() ? tableHeader.getIdempotencyKey() : null))) { + throw new OperationDoneException(); + } + tableContext.set(new AppendRecordTableContext(tableHeader, icebergCatalog, table, new AppendFilesWriter(recordWriterFactory, avroSchema, table))); + return Optional.empty(); + } else if (record.getCommandCase() == AppendRecordsStreamRequest.CommandCase.RECORD) { + return Optional.of(record.getRecord()); + + } else { + throw new IllegalArgumentException("Expected TableHeader or Record"); + } + })).onFailure(throwable -> throwable instanceof OperationDoneException).recoverWithCompletion() + .filter(Optional::isPresent).map(Optional::get); + var resultUni = appendRecordStream.map(Unchecked.function(insertRecord -> { + var appendRecordTableContext = tableContext.get(); + var appendFilesWriter = appendRecordTableContext.appendFilesWriter(); + try { + appendFilesWriter.writeAvroBytesRecord(insertRecord.getRecord().toByteArray()); + } catch (IOException e) { + Log.errorf(e, "Error while converting record"); + throw new UncheckedIOException(e); + } return true; - } + })).filter(obj -> !obj).collect().asList().map(Unchecked.function(ignored -> { + var appendRecordTableContext = tableContext.get(); + var tableHeader = appendRecordTableContext.tableHeader(); + var table = appendRecordTableContext.table(); + var tableInfo = tableHeader.getTableInfo(); + var idempotencyKey = Optional.ofNullable(tableHeader.hasIdempotencyKey() ? tableHeader.getIdempotencyKey() : null); + var appendFilesWriter = appendRecordTableContext.appendFilesWriter(); + var dataFiles = appendFilesWriter.complete(); + return commitDataFilesToTableWithLock(table, tableInfo, idempotencyKey, dataFiles); + })); + return resultUni.map(success -> AppendRecordsStreamResponse.newBuilder().setSuccess(success).build()); + } - Log.infof("Converting append records to data files for table %s", table.name()); - var dataFiles = getAppendDataFiles(avroSchema, table, recordStream); + + private boolean commitDataFilesToTableWithLock(Table table, TableInfo tableInfo, Optional idempotencyKey, DataFile[] dataFiles) { var recordCount = Arrays.stream(dataFiles).map(ContentFile::recordCount).reduce(0L, Long::sum); Log.infof("Converted %d records to %d data files for table %s", recordCount, dataFiles.length, table.name()); - var lockKey = List.of(tableInfo.getIcebergCatalog().toString(), tableInfo.getNamespaceList(), tableInfo.getTableName()); Log.infof("Will now acquire lock for table %s by idempotency key %s for lockHashCode: %d", table.name(), idempotencyKey.orElse(""), lockKey.hashCode()); var lock = lockManager.newLock(lockKey); + var lockStopWatch = Stopwatch.createStarted(); lock.lock(); try { - Log.infof("Acquired lock for table %s by idempotency key %s", table.name(), idempotencyKey.orElse("")); + Log.infof("Acquired lock for table %s in %d ms by idempotency key %s", table.name(), lockStopWatch.elapsed(TimeUnit.MILLISECONDS), idempotencyKey.orElse("")); Log.infof("Will now refresh table %s", table.name()); table.refresh(); if (isAppendAlreadyDone(table, idempotencyKey)) { @@ -192,6 +247,19 @@ public boolean appendRecords(TableInfo tableInfo, String avroSchema, Stream")); } + } + + public boolean appendRecords(TableInfo tableInfo, String avroSchema, Stream recordStream, Optional idempotencyKey) { + var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); + var table = icebergCatalog.loadTable(getTableIdentifier(tableInfo)); + + if (isAppendAlreadyDone(table, idempotencyKey)) { + return true; + } + + Log.infof("Converting append records to data files for table %s", table.name()); + var dataFiles = getAppendDataFiles(avroSchema, table, recordStream); + return commitDataFilesToTableWithLock(table, tableInfo, idempotencyKey, dataFiles); } @@ -244,7 +312,6 @@ public long processTableCountRequest(CountRecordRequest request) { } public boolean insertChanges(TableInfo tableInfo, String avroSchema, List recordChanges, Optional branchOptions) { - // TODO this is for CDC, will be done later var icebergCatalog = catalogLoader.loadCatalog(tableInfo.getIcebergCatalog()); var table = icebergCatalog.loadTable(getTableIdentifier(tableInfo)); if (branchOptions.isPresent()) { diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/AppendFilesWriter.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/AppendFilesWriter.java new file mode 100644 index 0000000000..64388f7bd9 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/writer/AppendFilesWriter.java @@ -0,0 +1,36 @@ +package io.peerdb.flow.jvm.iceberg.writer; + + +import io.peerdb.flow.jvm.iceberg.avro.AvroIcebergRecordConverter; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.io.TaskWriter; + +import java.io.IOException; + +public class AppendFilesWriter implements AutoCloseable { + + private final TaskWriter writer; + private final AvroIcebergRecordConverter converter; + + public AppendFilesWriter(RecordWriterFactory factory, String avroSchema, Table table) { + this.writer = factory.createRecordWriter(table); + this.converter = new AvroIcebergRecordConverter(avroSchema, table.schema(), table.name()); + } + + public void writeAvroBytesRecord(byte[] avroBytes) throws IOException { + var record = converter.toIcebergRecord(avroBytes); + writer.write(record); + } + + + public DataFile[] complete() throws IOException { + return writer.complete().dataFiles(); + } + + @Override + public void close() throws IOException { + writer.close(); + } +} diff --git a/flow-jvm/src/main/resources/application.yaml b/flow-jvm/src/main/resources/application.yaml index c73efe1bb5..4c6b7ffe61 100644 --- a/flow-jvm/src/main/resources/application.yaml +++ b/flow-jvm/src/main/resources/application.yaml @@ -16,7 +16,7 @@ quarkus: "io.peerdb": level: ${PEERDB_LOG_LEVEL:INFO} # This is build time - min-level: ${PEERDB_MIN_LOG_LEVEL:DEBUG} + min-level: ${PEERDB_MIN_LOG_LEVEL:INFO} console: level: ${PEERDB_LOG_LEVEL:INFO} diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go index 3cbcf1dfa7..4da8c44c55 100644 --- a/flow/connectors/iceberg/qrep.go +++ b/flow/connectors/iceberg/qrep.go @@ -3,6 +3,7 @@ package iceberg import ( "context" "fmt" + "github.com/PeerDB-io/peer-flow/peerdbenv/features" "log/slog" "time" @@ -20,7 +21,10 @@ func (c *IcebergConnector) SyncQRepRecords( partition *protos.QRepPartition, stream *model.QRecordStream, ) (int, error) { - return c.streamRecords(ctx, config, partition, stream) + if features.IcebergFeatureStreamingEnabled(ctx) { + return c.streamRecords(ctx, config, partition, stream) + } + return c.sendRecordsJoined(ctx, config, partition, stream) } //nolint:unused @@ -30,6 +34,7 @@ func (c *IcebergConnector) sendRecordsJoined( partition *protos.QRepPartition, stream *model.QRecordStream, ) (int, error) { + c.logger.Info("[iceberg qrep.go]:sending records joined") schema := stream.Schema() schema.Fields = addPeerMetaColumns(schema.Fields, config.SoftDeleteColName, config.SyncedAtColName) @@ -70,6 +75,8 @@ func (c *IcebergConnector) sendRecordsJoined( if err != nil { return 0, fmt.Errorf("failed to convert Avro map to binary: %w", err) } + // TODO remove this log + binaryRecords = append(binaryRecords, &protos.InsertRecord{ Record: native, }) @@ -111,6 +118,7 @@ func (c *IcebergConnector) streamRecords( partition *protos.QRepPartition, stream *model.QRecordStream, ) (int, error) { + c.logger.Info("[iceberg qrep.go]:streaming records") schema := stream.Schema() schema.Fields = addPeerMetaColumns(schema.Fields, config.SoftDeleteColName, config.SyncedAtColName) @@ -192,11 +200,11 @@ func (c *IcebergConnector) streamRecords( recordCount++ } + c.logger.Info("closing record stream") appendRecordsStreamResponse, err := recordStream.CloseAndRecv() if err != nil { return 0, err } - logger.LoggerFromCtx(ctx).Info("AppendRecordsResponse", slog.Any("response", appendRecordsStreamResponse.Success)) err = c.PostgresMetadata.FinishQRepPartition(ctx, partition, config.FlowJobName, time.Now()) diff --git a/flow/peerdbenv/features/iceberg.go b/flow/peerdbenv/features/iceberg.go new file mode 100644 index 0000000000..3fb839f738 --- /dev/null +++ b/flow/peerdbenv/features/iceberg.go @@ -0,0 +1,20 @@ +package features + +import ( + "context" + "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/peerdbenv" + "log/slog" + "strconv" +) + +func IcebergFeatureStreamingEnabled(ctx context.Context) bool { + strValue := peerdbenv.GetEnvString("ICEBERG_FEATURE_STREAMING_ENABLED", "false") + value, err := strconv.ParseBool(strValue) + if err != nil { + // only log and return false + logger.LoggerFromCtx(ctx).Error("Failed to get ICEBERG_FEATURE_STREAMING_ENABLED", slog.Any("error", err)) + return false + } + return value +} diff --git a/ui/app/api/peers/getTruePeer.ts b/ui/app/api/peers/getTruePeer.ts index c8b5c55a37..56c77e5387 100644 --- a/ui/app/api/peers/getTruePeer.ts +++ b/ui/app/api/peers/getTruePeer.ts @@ -5,6 +5,7 @@ import { ElasticsearchConfig, EventHubConfig, EventHubGroupConfig, + IcebergConfig, KafkaConfig, MySqlConfig, Peer, @@ -33,7 +34,8 @@ export const getTruePeer = (peer: CatalogPeer) => { | S3Config | SnowflakeConfig | SqlServerConfig - | ElasticsearchConfig; + | ElasticsearchConfig + | IcebergConfig; switch (peer.type) { case 0: config = BigqueryConfig.decode(options); @@ -79,6 +81,10 @@ export const getTruePeer = (peer: CatalogPeer) => { config = ElasticsearchConfig.decode(options); newPeer.elasticsearchConfig = config; break; + case 13: + config = IcebergConfig.decode(options); + newPeer.icebergConfig = config; + break; default: return newPeer; } diff --git a/ui/app/api/peers/route.ts b/ui/app/api/peers/route.ts index 4f468d8fb2..0d1d4b2ddc 100644 --- a/ui/app/api/peers/route.ts +++ b/ui/app/api/peers/route.ts @@ -12,6 +12,7 @@ import { DBType, ElasticsearchConfig, EventHubGroupConfig, + IcebergConfig, KafkaConfig, Peer, PostgresConfig, @@ -91,6 +92,12 @@ const constructPeer = ( type: DBType.ELASTICSEARCH, elasticsearchConfig: config as ElasticsearchConfig, }; + case 'ICEBERG': + return { + name, + type: DBType.ICEBERG, + icebergConfig: config as IcebergConfig, + }; default: return; } diff --git a/ui/app/dto/PeersDTO.ts b/ui/app/dto/PeersDTO.ts index abc3bc80fb..40c3466363 100644 --- a/ui/app/dto/PeersDTO.ts +++ b/ui/app/dto/PeersDTO.ts @@ -4,6 +4,7 @@ import { ElasticsearchConfig, EventHubConfig, EventHubGroupConfig, + IcebergConfig, KafkaConfig, PostgresConfig, PubSubConfig, @@ -53,7 +54,8 @@ export type PeerConfig = | PubSubConfig | EventHubConfig | EventHubGroupConfig - | ElasticsearchConfig; + | ElasticsearchConfig + | IcebergConfig; export type CatalogPeer = { id: number; name: string; diff --git a/ui/app/peers/create/[peerType]/handlers.ts b/ui/app/peers/create/[peerType]/handlers.ts index 4778182445..64a6b9395c 100644 --- a/ui/app/peers/create/[peerType]/handlers.ts +++ b/ui/app/peers/create/[peerType]/handlers.ts @@ -10,6 +10,7 @@ import { chSchema, ehGroupSchema, esSchema, + iceSchema, kaSchema, peerNameSchema, pgSchema, @@ -77,10 +78,14 @@ const validateFields = ( case 'ELASTICSEARCH': const esConfig = esSchema.safeParse(config); if (!esConfig.success) { - console.log(esConfig.error); validationErr = esConfig.error.issues[0].message; } break; + case 'ICEBERG': + const icebergConfig = iceSchema.safeParse(config); + if (!icebergConfig.success) + validationErr = icebergConfig.error.issues[0].message; + break; default: validationErr = 'Unsupported peer type ' + type; } diff --git a/ui/app/peers/create/[peerType]/helpers/ice.ts b/ui/app/peers/create/[peerType]/helpers/ice.ts new file mode 100644 index 0000000000..fc66a31b0e --- /dev/null +++ b/ui/app/peers/create/[peerType]/helpers/ice.ts @@ -0,0 +1,339 @@ +import { PeerSetter } from '@/app/dto/PeersDTO'; +import { + CommonIcebergCatalog, + IcebergCatalog, + IcebergConfig, + IcebergIOConfig, + IcebergS3IoConfig, + JdbcIcebergCatalog, +} from '@/grpc_generated/peers'; +import { PeerSetting } from './common'; + +export const CommonConfigSettings: PeerSetting[] = [ + { + label: 'Catalog name', + stateHandler: (value, setter) => + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentCatalogConfig = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const newCatalogConfig = { + ...currentCatalogConfig, + commonConfig: { + ...(currentCatalogConfig.commonConfig ?? blankCommonConfig), + name: value as string, + }, + }; + return { ...curr, catalogConfig: newCatalogConfig }; + }), + tips: 'Name for the Iceberg Catalog (should be the same as used by the querying engine)', + helpfulLink: + 'https://iceberg.apache.org/docs/1.5.2/configuration/?h=catalog#catalog-properties', + }, + { + label: 'URI', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentCatalogConfig = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const newCatalogConfig: IcebergCatalog = { + ...currentCatalogConfig, + commonConfig: { + ...(currentCatalogConfig.commonConfig ?? blankCommonConfig), + uri: (value as string) || '', + }, + }; + return { ...curr, catalogConfig: newCatalogConfig }; + }); + }, + tips: 'URI of the catalog (eg thrift://hive-host:9083)', + }, + { + label: 'Warehouse location', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentCatalogConfig = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const newCatalogConfig: IcebergCatalog = { + ...currentCatalogConfig, + commonConfig: { + ...(currentCatalogConfig.commonConfig ?? blankCommonConfig), + warehouseLocation: value as string, + }, + }; + return { ...curr, catalogConfig: newCatalogConfig }; + }); + }, + tips: 'URI to the warehouse location (eg s3://mybucket/mypath/subpath)', + }, + { + label: 'Client pool size', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const newCatalogConfig: IcebergCatalog = { + ...currentIcebergCatalog, + commonConfig: { + ...(currentIcebergCatalog.commonConfig ?? blankCommonConfig), + clientPoolSize: parseInt(value as string), + }, + }; + return { ...curr, catalogConfig: newCatalogConfig }; + }); + }, + tips: 'Number of clients to keep in the pool', + type: 'number', + optional: true, + }, +]; + +export const FileIoSettings: PeerSetting[] = [ + { + label: 'Access Key ID', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const currentIoConfig = currentIcebergCatalog.ioConfig ?? blankIoConfig; + const newFileIoConfig: IcebergCatalog = { + ...currentIcebergCatalog, + ioConfig: { + ...currentIoConfig, + s3: { + ...(currentIoConfig.s3 ?? blankS3IcebergConfig), + accessKeyId: value as string, + }, + }, + }; + return { ...curr, catalogConfig: newFileIoConfig }; + }); + }, + tips: 'The AWS access key ID associated with your account.', + helpfulLink: + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html', + }, + { + label: 'Secret Access Key', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const currentIoConfig = currentIcebergCatalog.ioConfig ?? blankIoConfig; + const newFileIoConfig: IcebergCatalog = { + ...currentIcebergCatalog, + ioConfig: { + ...currentIoConfig, + s3: { + ...(currentIoConfig.s3 ?? blankS3IcebergConfig), + secretAccessKey: value as string, + }, + }, + }; + return { ...curr, catalogConfig: newFileIoConfig }; + }); + }, + tips: 'The AWS secret access key associated with your account.', + helpfulLink: + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html', + }, + { + label: 'Endpoint', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const currentIoConfig = currentIcebergCatalog.ioConfig ?? blankIoConfig; + const newFileIoConfig: IcebergCatalog = { + ...currentIcebergCatalog, + ioConfig: { + ...currentIoConfig, + s3: { + ...(currentIoConfig.s3 ?? blankS3IcebergConfig), + endpoint: value as string, + }, + }, + }; + return { ...curr, catalogConfig: newFileIoConfig }; + }); + }, + tips: 'The endpoint of your S3 bucket. This is optional.', + optional: true, + }, + { + label: 'Path style access', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const currentIoConfig = currentIcebergCatalog.ioConfig ?? blankIoConfig; + const newFileIoConfig: IcebergCatalog = { + ...currentIcebergCatalog, + ioConfig: { + ...currentIoConfig, + s3: { + ...(currentIoConfig.s3 ?? blankS3IcebergConfig), + pathStyleAccess: value as boolean, + }, + }, + }; + return { ...curr, catalogConfig: newFileIoConfig }; + }); + }, + type: 'switch', + tips: 'Set to true to use for services like MinIO. This is optional', + optional: true, + }, +]; + +export const JdbcConfigSettings: PeerSetting[] = [ + { + label: 'User', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const jdbcCatalog: IcebergCatalog = { + ...currentIcebergCatalog, + jdbc: { + ...(currentIcebergCatalog.jdbc ?? blankJdbcConfig), + user: value as string, + }, + }; + return { ...curr, catalogConfig: jdbcCatalog }; + }); + }, + tips: 'Username for the JDBC connection', + helpfulLink: 'https://iceberg.apache.org/docs/1.5.2/jdbc/', + }, + { + label: 'Password', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const jdbcCatalog: IcebergCatalog = { + ...currentIcebergCatalog, + jdbc: { + ...(currentIcebergCatalog.jdbc ?? blankJdbcConfig), + password: value as string, + }, + }; + return { ...curr, catalogConfig: jdbcCatalog }; + }); + }, + tips: 'Password for the JDBC connection', + type: 'password', + }, + { + label: 'Use SSL?', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const jdbcCatalog: IcebergCatalog = { + ...currentIcebergCatalog, + jdbc: { + ...(currentIcebergCatalog.jdbc ?? blankJdbcConfig), + useSsl: value as boolean, + }, + }; + return { ...curr, catalogConfig: jdbcCatalog }; + }); + }, + type: 'switch', + optional: true, + tips: 'To enables SSL for the JDBC connection', + }, + { + label: 'Verify server certificate?', + stateHandler: (value, setter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentIcebergCatalog = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const jdbcCatalog: IcebergCatalog = { + ...currentIcebergCatalog, + jdbc: { + ...(currentIcebergCatalog.jdbc ?? blankJdbcConfig), + verifyServerCertificate: value as boolean, + }, + }; + return { ...curr, catalogConfig: jdbcCatalog }; + }); + }, + type: 'switch', + optional: true, + tips: 'To verify the server certificate for the JDBC connection. This is optional', + }, +]; + +export const handleHiveSelection = (setter: PeerSetter) => { + setter((curr) => { + const currentIcebergConfig = curr as IcebergConfig; + const currentCatalogConfig = + currentIcebergConfig.catalogConfig ?? blankCatalogConfig; + const newCatalogConfig: IcebergCatalog = { + ...currentCatalogConfig, + jdbc: undefined, + hive: {}, + }; + return { ...curr, catalogConfig: newCatalogConfig }; + }); +}; + +const blankCommonConfig: CommonIcebergCatalog = { + name: '', + uri: '', + warehouseLocation: '', + clientPoolSize: undefined, + hadoopProperties: {}, +}; + +const blankS3IcebergConfig: IcebergS3IoConfig = { + accessKeyId: '', + secretAccessKey: '', + endpoint: '', + pathStyleAccess: false, +}; + +const blankIoConfig: IcebergIOConfig = { + s3: { + accessKeyId: '', + secretAccessKey: '', + endpoint: '', + pathStyleAccess: false, + }, +}; + +const blankJdbcConfig: JdbcIcebergCatalog = { + user: '', + password: '', + useSsl: false, + verifyServerCertificate: false, +}; + +const blankCatalogConfig: IcebergCatalog = { + commonConfig: blankCommonConfig, + ioConfig: blankIoConfig, + hive: undefined, + hadoop: undefined, + rest: undefined, + glue: undefined, + jdbc: undefined, + nessie: undefined, +}; + +export const blankIcebergConfig: IcebergConfig = { + catalogConfig: blankCatalogConfig, +}; diff --git a/ui/app/peers/create/[peerType]/page.tsx b/ui/app/peers/create/[peerType]/page.tsx index d0de7cab07..c9647c0eba 100644 --- a/ui/app/peers/create/[peerType]/page.tsx +++ b/ui/app/peers/create/[peerType]/page.tsx @@ -13,9 +13,11 @@ import { notifyErr } from '@/app/utils/notify'; import TitleCase from '@/app/utils/titlecase'; import ElasticsearchConfigForm from '@/components/PeerForms/ElasticsearchConfigForm'; import EventhubsForm from '@/components/PeerForms/Eventhubs/EventhubGroupConfig'; +import IcebergConfigForm from '@/components/PeerForms/Iceberg/IcebergConfig'; import { ElasticsearchConfig, EventHubGroupConfig, + IcebergConfig, } from '@/grpc_generated/peers'; import { Button } from '@/lib/Button'; import { ButtonGroup } from '@/lib/ButtonGroup'; @@ -99,6 +101,13 @@ export default function CreateConfig({ setter={setConfig} /> ); + case 'ICEBERG': + return ( + + ); default: return <>; } diff --git a/ui/app/peers/create/[peerType]/schema.ts b/ui/app/peers/create/[peerType]/schema.ts index 498613edb4..3ece4a1b2a 100644 --- a/ui/app/peers/create/[peerType]/schema.ts +++ b/ui/app/peers/create/[peerType]/schema.ts @@ -508,3 +508,81 @@ export const esSchema = z message: 'Authentication info not valid', } ); + +const CommonIcebergCatalogSchema = z.object({ + name: z + .string({ + required_error: 'Catalog name is required', + invalid_type_error: 'Catalog name must be a string', + }) + .min(1, { message: 'Catalog name must be non-empty' }), + uri: z + .string({ + required_error: 'Catalog URI is required', + invalid_type_error: 'Catalog URI must be a string', + }) + .url('URI must be of URI format') + .min(1, { message: 'Catalog URI must be non-empty' }), + warehouseLocation: z + .string({ + required_error: 'Warehouse location is required', + invalid_type_error: 'Warehouse location must be a string', + }) + .url('Warehouse location must be of URI format') + .min(1, { message: 'Warehouse location must be non-empty' }), + clientPoolSize: z.number().optional(), + hadoopProperties: z.record(z.string()).optional(), +}); + +const IcebergS3IoConfigSchema = z.object({ + accessKeyId: z + .string({ + required_error: 'Access key ID is required', + invalid_type_error: 'Access key ID must be a string', + }) + .min(1, { message: 'Access key ID must be non-empty' }), + secretAccessKey: z + .string({ + required_error: 'Secret access key is required', + invalid_type_error: 'Secret access key must be a string', + }) + .min(1, { message: 'Secret access key must be non-empty' }), + endpoint: z.string().optional(), + pathStyleAccess: z.boolean(), +}); + +const IcebergIOConfigSchema = z.object({ + s3: IcebergS3IoConfigSchema, +}); + +const JdbcIcebergCatalogSchema = z.object({ + user: z + .string({ + required_error: 'User is required', + invalid_type_error: 'User must be a string', + }) + .min(1, { message: 'User must be non-empty' }), + password: z + .string({ + required_error: 'Password is required', + invalid_type_error: 'Password must be a string', + }) + .min(1, { message: 'Password must be non-empty' }), + useSsl: z.boolean(), + verifyServerCertificate: z.boolean(), +}); + +const IcebergCatalogSchema = z.object({ + commonConfig: CommonIcebergCatalogSchema, + ioConfig: IcebergIOConfigSchema, + hive: z.unknown().optional(), + hadoop: z.unknown().optional(), + rest: z.unknown().optional(), + glue: z.unknown().optional(), + jdbc: JdbcIcebergCatalogSchema.optional(), + nessie: z.unknown().optional(), +}); + +export const iceSchema = z.object({ + catalogConfig: IcebergCatalogSchema, +}); diff --git a/ui/components/PeerComponent.tsx b/ui/components/PeerComponent.tsx index 5689ea845f..cb1602072b 100644 --- a/ui/components/PeerComponent.tsx +++ b/ui/components/PeerComponent.tsx @@ -50,6 +50,9 @@ export const DBTypeToImageMapping = (peerType: DBType | string) => { case DBType.ELASTICSEARCH: case 'ELASTICSEARCH': return '/svgs/elasticsearch.svg'; + case DBType.ICEBERG: + case 'ICEBERG': + return '/images/iceberg.png'; default: return '/svgs/pg.svg'; } diff --git a/ui/components/PeerForms/Iceberg/IcebergConfig.tsx b/ui/components/PeerForms/Iceberg/IcebergConfig.tsx new file mode 100644 index 0000000000..55ade7b5fa --- /dev/null +++ b/ui/components/PeerForms/Iceberg/IcebergConfig.tsx @@ -0,0 +1,267 @@ +'use client'; +import { PeerSetter } from '@/app/dto/PeersDTO'; +import { + CommonConfigSettings, + FileIoSettings, + JdbcConfigSettings, + handleHiveSelection, +} from '@/app/peers/create/[peerType]/helpers/ice'; +import { InfoPopover } from '@/components/InfoPopover'; +import { IcebergConfig } from '@/grpc_generated/peers'; +import { Label } from '@/lib/Label'; +import { RowWithSelect, RowWithSwitch, RowWithTextField } from '@/lib/Layout'; +import { Switch } from '@/lib/Switch'; +import { TextField } from '@/lib/TextField'; +import { Tooltip } from '@/lib/Tooltip/Tooltip'; +import { useEffect, useState } from 'react'; +import ReactSelect from 'react-select'; + +interface IcebergConfigProps { + icebergConfig: IcebergConfig; + setter: PeerSetter; +} + +const IcebergConfigForm = ({ icebergConfig, setter }: IcebergConfigProps) => { + const specificCatalogOptions = [ + { value: 'hive', label: 'Hive' }, + { value: 'jdbc', label: 'JDBC' }, + ]; + + const [specificCatalog, setSpecificCatalog] = useState<'hive' | 'jdbc'>( + 'jdbc' + ); + + useEffect(() => { + if (specificCatalog === 'hive') { + handleHiveSelection(setter); + } + }, [specificCatalog, setter]); + + return ( +
+
+ + {CommonConfigSettings.map((setting) => ( + + {setting.label} + {!setting.optional && ( + + + + )} + + } + action={ +
+ setting.stateHandler(e.target.value, setter)} + /> + {setting.tips && ( + + )} +
+ } + /> + ))} +
+ +
+ + {FileIoSettings.map((setting) => + setting.type == 'switch' ? ( + {setting.label}} + action={ +
+ + setting.stateHandler(checked, setter) + } + /> + {setting.tips && ( + + )} +
+ } + /> + ) : ( + + {setting.label} + {!setting.optional && ( + + + + )} + + } + action={ +
+ + setting.stateHandler(e.target.value, setter) + } + /> + {setting.tips && ( + + )} +
+ } + /> + ) + )} +
+ +
+ + Choose specific catalog + + + + + } + action={ + option.value === specificCatalog + )} + onChange={(option) => { + if (option) { + setSpecificCatalog(option.value as 'hive' | 'jdbc'); + } + }} + /> + } + /> + + {specificCatalog === 'jdbc' && ( +
+ + {JdbcConfigSettings.map((setting) => + setting.type == 'switch' ? ( + {setting.label}} + action={ +
+ + setting.stateHandler(checked, setter) + } + /> + {setting.tips && ( + + )} +
+ } + /> + ) : ( + + {setting.label} + {!setting.optional && ( + + + + )} + + } + action={ +
+ + setting.stateHandler(e.target.value, setter) + } + /> + {setting.tips && ( + + )} +
+ } + /> + ) + )} +
+ )} +
+
+ ); +}; + +export default IcebergConfigForm; diff --git a/ui/components/SelectSource.tsx b/ui/components/SelectSource.tsx index 7cbe9d8b7e..ef30631cd2 100644 --- a/ui/components/SelectSource.tsx +++ b/ui/components/SelectSource.tsx @@ -42,7 +42,15 @@ const dbTypes = [ 'TEMBO', 'CRUNCHY POSTGRES', ], - ['Warehouses', 'SNOWFLAKE', 'BIGQUERY', 'S3', 'CLICKHOUSE', 'ELASTICSEARCH'], + [ + 'Warehouses', + 'SNOWFLAKE', + 'BIGQUERY', + 'S3', + 'CLICKHOUSE', + 'ELASTICSEARCH', + 'ICEBERG', + ], ['Queues', 'REDPANDA', 'CONFLUENT', 'KAFKA', 'EVENTHUBS', 'PUBSUB'], ]; diff --git a/ui/public/images/iceberg.png b/ui/public/images/iceberg.png new file mode 100644 index 0000000000000000000000000000000000000000..e4a99c3951e6aa99d6e44575f236f32aeb5710eb GIT binary patch literal 17608 zcmbXKcR1Dm`v8ugWAA+sSqbGFl*q`IGD68X_90}?j0h*QC`5?t%yW+8WN!!ADI+Un zWJN-e6~2%2dVj9#`}^nj&)0Qzb)Cn!_kBOseLtVyHMq@4$3q8!Kp1tjZyG@$WJuCK z8Yhxv?{CDn=7V^=3FQ-R=cD{}fbq6ndM?oDAyN8ZOj&=@@efk`gAds7`Iycpg zgTDTpolbl`k+HR7vRa(aTpw}_`CIQT$}8D@{hotMK}N4*wUiS+;tc)jZJXdq8t+?P zPY<^>&4N8@SJ$EN%J&n2$Y^i?q-omv5ZU$ex_V$syJ2Y zqPw+$iqQ!fz@S8bpK*`C=8yE4C8zw8sdI27O>47@hEgCP2~n3HH&3z_qv}lAY85rj z;vJzRl4$S8rd-(*x9i^+8hb%zmLfuG-Hk2Vxp+tCS+KRTQcJA2_PlFA6rNL$jD-N% z<($`*TIz0Z#N?N|suHb3UjPBlc7`0yZ5^bJ<-V9!zS?yK4M0 zFbYEjk090*^iao*J_Fd+t|4W=bV{h*5{DSNVggLuGK+vphynQ+{?IqN5hmF%Xk+qU zw@@PI1vMg>Vx&nUWzg#}2-O8Bd{!gtfdXjkpf{=9+^;RLpnRidLQjDZB%sE$HO4rw zP=|uDwJ|a%{q}XFPTo<2eKT27dKyLf+K$R}Ea1vBR=gGpy`g_QG37-tiE}to=S}eK ztM*WW^)*@x{WLNP1o0bc?C!1^6i+2_R1J_J7)*`cyApWNqZSzJ;S0`B(UdGW2un5K zkH!=-BUr-d=zUp?S)E7*e$rR~EEI5vr}O1nrvIO&S_*hZ?J0jdof?<=Mi57}K@|rW zXi1H2bTz{jBN-ve&f-*Q5ok&!R@?~Ok?hHLNrM4D@f8WN_SpB z&chB9O&>)T^bL6rfD^}LYE+Y~g1u}?j*k3<`CGeSo<;fiHZ10`Gu9V*B_? zZEtQp+^Xc@=71p>cyNWTBGXTAn=?ZAT`p(Ns}nU07UJNZkb?wxpc+xbsXYz@Lnl>p zj3IWP@gsJ5@FO+=4*KqwcRgaePWfEMgdqONIhU&(mole6+2nRRMz(CR~#P z>@l$69q?n*I0}eTe%aSC9H$fTj-i1>IV*qE{`STp?<6g%Ki*+oK3dD18)sitR~aDg@$DoRZg- zCIqF2Qo(B}pv3dF475m^D^@&DD-?KkkFQQn2n}9 z1{xmz$OOS3Uuh8FOn4AKl;|y@y7Q!07lM{6KHKbi^#DN*r9X_u2ExQBp~Tx+S1zK2 zO1wo(fEkcMnnm=tyjUt}kiDYFt1(bW2c3D3yqz{N;?TQ`n+ifh|KL0gQqT$tp)t36 za3}J{0>+@844v1d>Tr9GIiZr{@4JZQTlI=rQCJ^9eBM`O^HCByAP9(1Jem?%6gr8L zjB|kwqSg!i`u|URyth_88H7Z}|4<2{Sn&`@6oQ37148-#+fMSO|Fv5&ntj))SN!M- zfk^M(;;767SmV7ZvC7x1OpgQ2zU1h>>VnABHxddYKUb8F?J;t8z989=Bamwp9O+nU zeD`?Y_{+7XevH-90QHc5$5?cOi2rdrPrQ2X1ynv3y*IGddC#;%pIuXA@29uODGjl? zX~1p0<7aX(;g;;c-CjkOXq*D@`Q_mHqbo;Ox^(Nc#Ep&0lH*d1@el_+jJX+?>Dcn2 zBd4Zn8LEW@wc$MeTf9p1pl-cpFs86h7h~d1PIlN-SBXDWtm6o$AW4kY?EY{pvpKDt z_UKX7@B>``bwQMH=KRYy31s6})1IQ2jp~!bu(p+K0V(Be4`bl2XP%kkP%dr5@vbJB zc6K1b{1`aZa|_RA)1xq67Ier(l+I8Atqm{LD>ed-*^CX-dckTcw%DI2RcUg`GN#g> zM&jEGjd2$M+tIDwemvFD*@F7tE%xT?9|6hcyo5LYj;xvEPYFX566<;(Su4c;P^{rUwY8sGf-;V`oa8_h##(4Rw zZvcKM!-aCEVTw6+15P%AG_#r2eAd%y7wZn#G#WxBqHHcbV(0l>^nib8*MQiMu+kwE zl+6mdSfuCYvEd+UaxM{VER+%$bL$&?lsGwaDzdabZ>{@HLNev#y=?m1%I|(;qa6`e zFU1xxpIp~8M-|aZBxFyz=@rq2MJLt3rutsFEqp^=uDwl(ZSodrUuQS9 z_a3`sd@}ms3OzLF*T~Q0s_^~gOI8wipMGy@F_PN`n#YvG%p>?ZO)|&x+x?ob2r>xP zB#sipd?Q5cvI-hKCP!;_$8|ytW5%xeQ{m`n51-kn;>knWlYo7=mp4Zvj{DFv&wSa=C7i(A+QoP}e6p z_O2%cxS|`@TSSUQ(r4VcP(1SZ%@-!3{utSuXvmdR4iLzLBckBCSv<;hO&u2?-urJI z@|C&hXTt>?u}SE?XKR!@we}YyO^quc2WW9qL-~uhh`|@Ft|qrb00~b7VYmyzyuriT zahwzgf@~=-U`D0vz5XONFuB!<%(OHa>hf`Gh{Q?m6?W`HqiUL?A>>d6cP-UmUN9w5 zZpLsjJ+F>VGrZ$0vi)*KU%M*g zbHuDT!|xk*$;DGVGUyxB&;$2p9vxx5rVAWUI71eX3{b)to9Zld+bsH+_IS_6nOr-P z9;p-Oxot6#2q>9@EhcHfu*~fqFseOqosnzl_juIy=+rK~c|KP6Zg_uz3OZo8b-9-6 zB_L6dqew$2EKjO^#stZxN@y_?~vx&pYHq%EZ+X8U>9M{t=z(i^=LJ;vuS_J^j>{3 zYR}JRV1j}cSiu1vlTD_Yu zt;-(9er?x1!pX}rEk%lttE~@rP7maTR-mr_>1X-Tq`0jV-Hsar^cZpAy89{Mf?grw zMfJ(itt~0lzgY-Ytjcy)EAB&X6+a)aN?#Nsvz;_~@peK42_^I(Uy6^Tn-MWV=LpHl z){=hLZ(_`3>K^WPXUpJ^gD$XacU>x>rS_L``5}Z-Q0sslo5h?zYQGqd`W;`H_vKpx z`%@V_c|eMU#+3L6wPvjSV=h8+%S_$l^2~eKv@dXK7f(4DZiM}*Av+FIGW1i0&^f%n zb(xIwvOE&7C(2|#bU5nO<#t~Wf)Tg}Q2Z@sJab8U(YFmK z$h>(?a8GsWe>+k4*U@pl+>GgjO)N5a8a9MpxXUv^Yl>E_SG*mcdu-J8TNEWCywBTw zeb7#(6YbTi^Zn>l2tZH~=cJ3mwF4LhV?8BwoGoTleKqpN6-SRdJ>>9y7g%V>r=u+H z9A0_yB_5rcrBwPSQ)aa$J@)}?u;9y$AJ5fXti+H|yVBIS9ddHN*5#63_NUq2Wqw}m z;vQD3Hw9TlF-Qw-*ualzWEL~BFIC>(CZ1~IO-kgAWcH`W3aoDiEvi}xGulNQ`w#3; z26dOeAiT(48BB+Y=8S@E}F75pz4?=W!B%h{z-Jg1`JVV<0F!wo^^gk z5n1$dBa4P&yZ*WLdAgAgQy1c@pzt!AldOpx1?l(M>n5v)MVsAw0=^emh)fQcraOT> z#H@1r2CMdhV8=IB@sR=F8)p~~$ayB9yQu2!I|%(OoV~(1La9wgPtm8u9GV$RAtZF| zB?Dj4lr_B=7Kt|{^)bDd(x(V1tIDEz-R%z~ImB9-7wu<)SS8~e!@4)~%>T`k!B50a zi;=llQRk6gdtdE?k6X5<)cbY5Fxc^~cDmv~t?lY_A`kf+sPo0aUm@I6dGYs6a2M!b z^tAZ)o_)ja-UJYU3eY+aWHHE08LP&3LuKf$?8p_MqLh$W1MTsNiD6^-sRe|E^HtJ4U;I{wHbTtks+K4pOf@AFr*j< zMkaVWB;$Ee3VRo4x;QfHyv)avZ`f*|Z1$%R+<}wjr7M-{x$pXJLdnMSlHX-8+T33` z@Gh%zKi=GwXVw0$2zh&D%9@wf%!{3{8x%j0VlHXBKJ)m(wo3#IqSrSYCDY{g-~^wx z2Zg&hyKdRU67$X^ng8QF6h1#=A zy*2VNe7^HWtGhSn|1)!M0bs@EmR*&~sH?~?S1jL7kN&5$ZT)iN7Xm|zrR9;g%R^&Q zDs8hT5)`i6tRwzkn;y8;UGO`UEHJ@ALB9jYn&42{M5mX@X4(!3unkwEJhaI3v?YsvCr=6Xd;#N z5{AH-Q8_*?tCGPF%%WBEeE1;OMM9A?hNXiXP&ay+y0?LA)F~Mhz>wy@(Y4b-Y8lOC zgWt=%$4t%E(s3^I3ua7u{6>|1%q^hZogVqIRl4^}(WjAm8v)N; znnD0OSlBWuq(1j9ms{p+IW2NH;=HL$9dA+9zHaS_BCw?TJt*YpOg+^@BO6F75jdB; z)sK2kG3EaQ4^b|R(5h;>ZtBkdm4mR>Cha^l%;54}QH<;aQ-RXQ#So7ApN%{BxX*sFxy||Rd!O_>^DeHGLfnDxpsFN2H5egOj>nBU7;0pwj zC-koH`sSu=Z)bnl4V#Ya-VHAE`}&`&^zx_7)|@l+jG>hS#|{SSNl62y z{@S_!i`!}{OP(DT&mT->E;ZgTWe9~nsU5A62Zop{iiDzJ&H)on;!`5@QYatU1ar&m z^LGp>`j-bl^nrxn$}6x%sW48B+!^Ux3SSIo0;S)d?5wBC;1h4Da)a$IB*5I<+C;AH ztx4e4oLQH(;pHxQHfx7;cg4gr?{Ah-UOO{yg(emY|Gow>qsWJJ@C%^Cx&?Y-+QN@z z7y4hkd+fo!V%n}-OU?XeW+NRKsTi3febP5^2w`AxgxvqnFZbP@uw%J`yC&y(gssle znTyAYo90+XG3NMP!=xtj@QIfxi;&J<@gL`G?*^{4g>d6WyhT#67le5j`W)X9vi^uM zu-g*p2iE^h`0Y^a>-q@e?O(ojcA`c%&se`n48b|*ZTm!jrCI35Ssk_b{b;wZFMYiF z>GKBxv+>l%QJ4{OVQm zTY_cKn#EYRToo{@9JxU=@ebt=r$hcu`i3}%uy~Nh2jQkQt=j+9y`US`lB(gzwIF_@ z(k(!7;0$%O$B-i?H0;;c; zfWb<1b~Bl2qU9Qiv;D3bM*qF`m`0B1@}_%37GiEtwYhe2{u;uSHV=Yp z_^Jd}!#^r|AZfq+QV7oXI zKvAjV4pMY+RAa!CDo+{iup8;mrYm#waD^edX{`%4}DrL}Xj zWaQ`jVkEt?VC2ik3`n*T5+l3u5c%WyS3eFO?^yZP^zSBn$9Q3!NC};r$+eK+krENJ zof-3~yTZ#FPx>B3kM=?k;Qq*?Wpxy79Rqi=HZOnG6<)YDEf!U-Q{On$(;H)>L%HE8 zZKx;Ob#9K4O=w#h6zYSu<3udLy7f~X)y>>aH|3ara=kM{Yjgq1C z2)ef=l#u4v{|MNPr|P|^lW;U?cip!7s?Ia}`QhEJV5_esdv%n+6dZ*4YPKh^)$2pmdt8{X_0+T5O4_;O%C*d|O7DJUxQ+xA`;rXwcl7XMB)CC4 zsz8V==oU9MsR5G~o!HUN<$S>3aou#8*YXlD2^WI2vTHt;1U^?Livf90+I@RwYVgZR zRB>Ub1y@Q4vl)#AdZK;Zo-S$rRrx&kC|*o|cE-Ni8b(P*amNd2(nOpw@y(j)T`{Yx0u8$Er;JYqRgD10+%6HQrwhlVR zm>4bqr~0NCwD{tcM@K<>o44}Ol~4Cqsmkmh|4G?yG`gm@sUmR*7)IqD9@F~}8kPJc z=GdR!2upVuH-XI1A>&3P4PIp+YZKkqQbC+Z*$Bc*`S3BLA|HL}kgM~YWF0&U97DNxGmSJ#$nw`^Q*^s$ld)$@jIN6qlz#)9@)9qFLmAwBl?C#@qTsh$GA zF4wKPFs29bK~AW!df^R)_!_*zvBH%NaiqT(Sz1BxG6INlv%C50{+*x_rzO1R<6h!O z%NSXxg<86k*C$NH+25MR$D75Yxzttoh~iLBF(|#^!S8=}lA!RK*H%5kp%xExOyX4} zP=8P&F!i)Yr+l@W==J~;VX|80`_}mVlCfR6ZckuI&__*t$ff3p!Hk|2k=q zlgToT1M|a4EB1^9e-!zFaz`WdmM-S)A`kg-izIJyNV2^{fx!3f^iq%Rq7pQBPr^1jPod-U15(75UmecT7P*Mw0$W!{jW=(zfxU#LjrZcE7viBus_s$ zJ%(r8rr@R!R-6ClV*YbFPGKG;;pr$LW)Qf)hgyuV*uIOUQJ{iXI3fM54N&8-#tTra zqct%uSzw`m6utKqejFj7gwcGMJN_*$ZcCNoj^1qv?Rq*%qzK7>){>Xr6#GN<-Yc(_ zOMRZZdv^4l3rVBDs8TMgz*wiA%lU#D?ggT1q`6t|<(@SrBx4S*s!r9ho7C2W?NX=g zKh=vN%;)|<7&Xq-44IJV41Twn$6ulNNQ)}5|52a;N2UE4cqF1SsZW*o4Xn!GU}yf~#c&Dix&q~;7L$<& zT7<7;RrtJEf6{*cX_EN(tK2W<&(4!fmtZVzvFmckhl>!X8FC~sFSC*{dYuwMp+S{+ zI*@3Ka)_i%OTNln7vJ~SF8Z-WtM+Y)k5<`2$jvBovmv)TJpu*SP_jV=9X`K0UIcG0 zuOeymNE-s@S<(u7ZzyxS*%144N6yFzxyMR!ik9l1y*qI?!=W4h7|GJ23k~73Dl;j@ z)lYto@?ej+2;4P84_g5p?kDPZzvu3W51eBH5!y2PV^`%w%h<~U_1v^sPn)K5`(~`T zBN||a5$DED2=ZFo+*FB*Y%xDZE7{(0QZbM^6fAz<8Ln+(nGuS-T`~LSq0sqPe_c*I zz)s5c>s>g5(&=0DjsYY~4N&y>^@8yD5H?z_BvJ*jdBM36G% z;AfxAaU_Yn-#Y~o(0GXsqRzV;0N&rA+;Qiwt&N}nI@6DYenAqp!?I~M0>u9*8Wwf; zxSG2rNs~13{4HX{bu+JzJ(0IF;$aCaN7YL=^RlQeA|1hcB8Upz(n@2~sHLSvF8ug8 znjDw*an!fPF#aBRU?Pc{jJ!_e2OZKshrSTHQDgS^YLLv`krLUe*X{!SEwLv102(PI zvEqD*IQV2I2J4HSOkedX4L{kTN8BV98oC8I3WtI1gY(qsMi8JWQMUmnUZnePgSyDB z`zn#QIXORDrm5}kZxyP<3**2wc|rxoeE2>Sx@{_|Ewl(D4@!+py6lhL!b>eoitLXm zZkAtYll1?0)R)4_<-~@f5^UTY1&goKjO~~n;VUM*tZL!y zsR)DuG09hH6P6H7R0A^8z1Ibk7*=pqkGN~0v;Na|-Tk0AoEALEe>?a$Q&S(|4{nG# z$;neNtd>#x)&jY;?8(abrlXFo<){2SBE==ba>5e$-)0P(Ky@Pw=F$SZSBGfY^#`k|%1aczZ#N6}qz z%m@wp7m&Y!zO8LLE)}-Nl{rU#F=Z;Z4EALRy>)A3TT-1K@`M)YhY)`B#ul0IcCPIx zCf%5(&`2cIbSL^#=IO`ZHnnunAU1+zmZEQ(W0Yj2{=tT!)X2YI4WFLoE5b0#^*{I^ z;5rpeE7NwYLlvV-SU=B=v2<~NHid*da%LmUi)7B7Br`FTJNK%<-8L*wy*6V4MVrZc zxuEo>?+#}}Z!j`!_WbNvuRUwB$`>~K0Rg(4BpcjHXl;0;aOKPE-`o#)NdoXF8r;e~ z`(er12|ZzA7}MeT*1*6+N(<>&UB!0|ZtXgnAH_(|6YtK}|F1n+rfnciNN7N^kt$~0 z;K}kmz)3wX!8t*_xbVk`s_H5G;1=hVt5=LS-p+WZ4&*`Qc?sOt<38$8f}N_-u!hPV zMustdGJr)b5wJf}ErfzLL?eFm6&xyCnlmtrJqL3fUZN?bhW_R7Ws_?j%l?@)`h1o< z@=H8jogAcTv=ki9BG}0_t3vDG-sfZ>t7N9}MNy2-VRFfOzI@ymb}(#hgc!U3h!QuT z#6_SS+&Y@Pa>ZCqGLmX2`YH3)do3C-&9q$ylJ1c&!oG1Mc>g!6cPaF>2>LTPL_DvU z%$LH3NC&q!KF!bc0a(n)IQ-{2!ve-)L=hoq5$9FtklZc8(pczH4f;8G;gaiWsl*>e&!(gp{&@XQP~Fd#+sCO1o>{T=dV2e&^r$TbIn z=me1gCWJ@lT`1fg?~~f9Smm0T&CF>8mBzC#d`UZk@J`*4kL$#Kik{T!YHNsq8}`gc`C;!xH#PS7VbT+7~Ld3~s4?sO+^ zcqNc3#?#a8^07)in+ZaY<7>d;qc$(O$DS-~K$0O0{&H;tYtawMxw+SMiOH`RNsOpZ z8=O~0648ZH8z5N3r>*P*2PV*ym?2a}_%D26HDJA<{fBBqqU6->Cq}OHvNkOG$f0(ba5-gN%6ITYeM5tnd z^JO|5htRRrD_mrpsJjpUG%+xQWI#Fq!~ zZZn)|eTNcpOBTYH!}GZEvPDN~6rv-o6NHfMm{06cqU7W0N+v{2c45n0ML0|C%qJwE z{2;IFq@{hx_B9>n(1JiN?~nDl5GD7%fD1S7{sopzeEijq9!lR!yf54$r>TcejybUF zNZSK2y}vomhUdLPr~Z5(JCYj)<3fv=W>AK;z+l5ZRzT*VwHeKC}(ND?~bwYI7h)@w#QBX9w3?ggcN!vp+U@h z&RUn5Ek|!D^1B_fMTo8!phwrdSmt>5U-a;cLPQ}F*`pde$0Y}fdp=9xY1ZW{ctx**e}2j&)QKaUwf*;c zlN(bRylt&*_t*Vi59Fl_;4am)vM%`{7S=ZosfSD2+Ufbt`wQn&%qihNIxjc~(Ezdq zY72$J>>P#p{pa=1jEL##?D(YgONkD?+maiBWr>enS;-7oGM`L{V>HSa^@r$oq@6op zzF$86EbpH%WayOx=Jv*M$Tt*kto`5Is*iNa8B{nB789=Y8ENWM$|Ls@ z8}U{h4k!~bwr;syuYs3-JCJcZw59J4eeS)M{>z20^)U4Vvw6$M?g&~FqXr9eF)Jat zazyvFrE`R}&WH(lb?u?`t=Y-BfxJ;~k?Baa1nCwN(Ld-^lYNyWE(4aY+|}p%ko-1O z`t=c7(%C&3X3CjKlwLj&Lmhdt#`mM+c-03=2J&dl2)$P0lsV00Af#=}0}h;R`(5Rd zn^>x!G%_5mw}^DW++RJmi>2etV-=&JwNN}5+LjEYm&D_DG9x9V+wfB974r_5WAhxZ zgP8TyNOi1L^9ET)jo7wTGGOMFjKYC;zL8_K^0cOC$R=wM*+2nU_ z_v51U0$pu?QC> z=#j@%5^tp<)4WA2Bhf{^y!5GVMEkcaD*3_7@9nQnF#Im-wQwzw7y`-qVvslk9E&jp=K6CIupXM07~XfW;n&f=kzIm%1D$87XSju5QZVj-y5De0ASx zM90gkKP$g)8j_IuD-ULvnSTETl}>46aH2Qm>JyMV1eRL)=j>6v!p>a}co_z}lkbA( zs#O0DSk}m$m^bVM{%eD7gE^Mc{Z5M zvRk~@d(FwlKNUYNC&ID>qcQ?lJNCJB17X!v^$cLOvzlj+2N!TJQlarqf%P!LeyV8BW$Bn>)JUaY_2%jw+Dmi9_rZi|JpxP>kn>@Q{$Z9Uwl~`G)r`9!Z>n5 z8z>|my9eYys!Lb@jF1Gw#L~zwp`t< z>OVs6!24|1|_LQ!moA2-hDH+m`!Mt3DzY_lbI(8gc77+M9BhwR%b8 z$3iCN&d+;u<#<;7Jnu$eeOv+pwCQG(_6o)xVHzXuO~QpOQJ+V za1S3Vw~GP+yi5~7lx>uIeaQ_!i*DR4p?3`ka2fSsLVy2OcOtbm!Xi*+AKL3iS36H3 zJNGF1WK^@G>etTJcIB zYxsPc+xTyeOMupFGbL(91|p^4b5Qyw^c+WimF0kX`wq?%l~_mi0t7^ePdN+eyio|UA+xmt+p2(rs}D*R=2F$>ey7R z0?)n`GzRRr?d=`vxdyBR?4OuSFJssFd8QXY295$3p`S@goLsgL z>pAVy^uM>kKIpb;$KB7t8C zH7UAm)fc_T;x@OVzdhO6wEyS*-2DV_<4&1rbUx>qCg;3RnN7>WqV3FHg*FD+YyD8f z?S`dE!&eyo(j%=l{N3f>5$=aYEhdV!XSFAn%?%jY@beWZ_|vZJ({q_Nh{?tApP{mC z1RPWIkmY!KPC50x?Mzz}*Xuu)Ev)XldT3ri3gLqngr-9X4$U1k@n&CA+rnUGcn8a& zQhOa(#w1g?!c<7=?}Z+I43eNxmx}WsPnYSh@G1?#{x&pjuo8rc|G*~4jeT8r1&P}B z9=YSu@odUdV8 z$YruC)|w*bimjQ=b|%UUKlcIxTc4?%U#X$Byz`6i6T)HhL#()28c4%38k}u@@CYc- z>+5o+IK!|nT?OB^5b7|Xc0XwMAZkL?-N!B9UMRTNv0$w%_1#1a?9iPQVKth9DGxRZ znHR)QefKsq6u=`I1&*rX-sFd3-SGhT3f~M2%B6{UXqPv@BWfUPp-N#4Z!>tf-FG0L z{R_E!5I1qz-TwanR2)-~*rXd~(okrpoK47a>kO`2z!-~IwFioY(gON%iJyl6{e@GZ zaTDV1-*v&HAG2{aM#m4*Oz`S(0;AvAB*gXs(Ko?Uq@R4enjs z!aKqXcq(oaQ9kfMv#hXRBps%wV(x!r~WJJv1lTrtd z9^+JOI$fGf*?^AP+iv)75z^DF0urvEYAjQV28W%P#E2wE&09mp%j=>jozi62RCmhY z_W^A9`j6+rZp;t#wHvgCkLmWkde2BmW}c3oP3G$g!Sb%#{4?CV!Rc?>FmenLF;=iH z5B?`Mzs}zQd$Qk}2t;|bcarhIo!p(v2rxNo+@J_Yx^yyH+w=)DdL|hz1q5+$XW4l(%t2;#IL^OIL_;J;dA#-!;`y=n)d0o zr+v2CUTX$%rm!ttwfQXQ&>j%IuC0XR*nT5Zn=67kWUtH|jvZ~rtM~d^malJdy5?KO z3!r-}hQB+|f0 zK3DnV#utT<8*|OsL-mc%vs|l~Z%Ju2a2G@QYa2)rOlUMXe}6WUPR_q;FEn4}R2jnY z`Ys&FR{8CvO4hPJvqqZbUsbt<;Tzr=PYbrK2@jIvo~rJA?2bqg+5j;U5yR|7)Xd_E z+K-s#xd<}I=@vYD75^NY5`^3yeMofL4ss(PS11{CDH~+9iNV~J-%Q+L9qRcpI935N z(Nq7Kuw) z9U#`q)yc*Idh0Gyr-63>r)jfneJmH^3Fp6YL8zObm-sILY2XxX;$NlEe0N##e9%_V znq?#fPT$*%1kZb?V^xE+*d_K^N0JIc+^|U^$K567v@Df*Qr0lz{O*tNpz@+gz>>eZ z%|&+@u>P`PWzUzc&pKgXRr`0m0?xGutvbZ7Hoq@CPDzCb4HyUu|FV+0f5~KuCu+;# zqGpPtAj+S&a;Zy0l{1Uk{J%@d3w4i7?nVna*|fVoOu2smvfw!?m$W+9sN^^Q_)RS4 zEvK9u=lBZS_G!mxJm(U6(&fnvbrXEyiT>k&9jZ%m46RQtZ|?^=`MG=#E|NpBUVZVg zDXfAH=EbHpA(PJe*E`~wR6*-2bW^S9;!+OlGd(ppHq~s5DkMNT`Gf)^#QcJ4%dt4YI2)(H>x`p{~ zvE!SEAAtw>Tg~}vJ!C=ybSdlsp6xKe@7!9`1=Nq8CkU3xpLXA)Kl|JdDeI4?Z&lQp zgFy!zdR=b`Bvi2pIxGwajv9dk{P~j6k6?x)YvodFhl?p$6z<>8aL*3B-00U!6AMJM z-&E}`!DOaR&cW0|^zRSE&(~5dl+??{giz5UGk&b~ENW3#XpE2i&SVdt8)ns$e3G`| zy(ImRXxQ^^E0BYo2AL6yODp)nmlKUk4F6*$rZN6Z1PW&|b7$-@9QmR$D9u9nO=qs( z^-V^}7o<#NB&CMbCvSK=gs^Ng>u*vZ;2LQcpBSmon4f!_#XWcxScIMPYcBdHkJ37z z(1GG45vgubI8?(l;CEIuEYabT_cV92=KUMc0bMU~1I~z~RBIw8?GLS4rR*h5E$CUt z81EfdF_gKO&c7ZGF?#jh{trtIDwdBZ@tsd|<*#ZJD-|93;!|#bp5MmJvPi_A0(9bn zYbLOu2A0Zt1L`P5=dFDQC5#ZrnqBv_Vb~sg2N)0ls~WALWz3ar2802;K&P3jVgLKL?|CKWw@MW5^FmbBrmR!;_v zPoj)EUs=5hKi7Bn(vCxfH>lncqNlm$uo>>D;Z-kgG7SXY0RvT32*IVKB3TsU%il}w z)T4j_PNDdLeD&V->9uFA^tr%lZ-YufZyKbwM%ur^b}kAUb7h`k7~#Kt-3l%F0v!cH zgLB@r_h_9rB(Ni1^MTSmK`T(<^tjia`*JgbdE4o3fnxZN+P|tHzV?LHvihZ1@sO4d)glr07=je0kQ>}GVF-qsW6|D6Cuj7FbcApeL#3vTOIfcYf5~ap?@vZ$` z6-@qrTl^?Es7W*T0mWa+pDK9-9h6-i3hS^Qbjue1Am@cu9nPM4YY`((lSFK-Kv|ty zFAv(CaeR^|DrI5)7Kz=Pi&{&d#%bVA_En46_ptAkq-*gLV{TYb*M5X|1; zLQh02qm2f6{$=Cu)JNzUuz~PH5OeSe7m8pqPhq#rWk4{!e4peU{jokc9k@$7KB)_b zpx~{nZQNE+0iIO#j9{%8W0zS3`aUJ4Ciut?x(Ze&?JeD~dVW9ByM5}Sm8>~k_!Wk3 zO|WfPQ{O#MO;HVXQ#$GichZ`Vx2)P9``1yaxZQCedgi8{eYPQ&xPs01!I%-UqcGu# z9VYQb+w?F*qLwY8=NqjIU|; zM1Pk4qZ?cS3E}%*C{)#+b38D9~y9TAtiAFTotlo~>@SpeT zjr-r}jJ`|tr|+gdx3e$vM-bbOg1|P%Iw5T88NViu#%f~fAneEQ7r})0Z`Wb&7f(#j zBx941aaXn_IZ7kNKAlfc`6nIx#hmc~<-0{N+JLW*&%!v#ufLLL|L#u9Mqn}RO|JUo z8@q7NlpIo>QcisEU#sU(@QtHwemE5J^$s>*IgWh>yzBB;PD)cilrJJ%K7(M2mT`FR zNCP=}pvOXiaObTCedup@Kq)Jz$K?{zt7Rz4^nU{?QYCcqk}+d0X=K^lGI9DMBAyNQ zu_|PcNvr!cC6(;>Z3=PZ4!@-&GXM9>L?}Kb24dAss}3B&f5`QsIu=yydZUdo zDkwaNDc(X3I9YB6U7Wy@L?I)uecUKYaqqoGVA znRaqQXCJ5z*Z&F3FFgt571lD$Imtqn^2)yI|CAurgCB5Lug6W9Ln?)z#%-#B8tXel z$v#Y#9>|BX5N_fRQp=;E#eP0WqdqoZEKUV5!SblLK$RZNq_33=h$6D9V{<6VL2<5H zV*C2VKx$A|?JiKwqca4m$|1GGAO}~5RJ5!%M5aanHLipU&VA22J8@Jj)eR)|kyu!x zx3~|g&oNazYToVZaD7jzQ}>K5FT3b`Rgbilox9FTh#DQe6BnXAlSu}ztb67+!ea==0GMd3SJ8ZAd;xc8w)U$ zyK$~CP-oxBWN9>blu=+tKdF1a=IwZyDKM(M)jW~!d?OMc{`-bN@?F`%XOtvy&{^(X zbnmN=RfI>VY5Tic)EJUgQqlmH-r4D`xD>tIiIOC**wCNbCmlZ}UQRF5aW@ACAbxt!aQ*qp21}x^sl*gQnXh7d(K7itgMI5qE9i`?%d1x?|T!p z)-E;J93>k=4K7$kd(D?l%$Mb*2k{sVAX=7y156-=)Z^yNsEXZOO}X?a(H)l_)z4R9 zd?aLTP+Zl>n6^rS|Gyk;D#7%(c6g}5;oj86oA;tYQU@7IU(ikIuchT4x&^{y+WYg`e4?p^(2HZucrT(7;GAbhHd^;x$ka{|~C-bCdu8 literal 0 HcmV?d00001 From 06c68e52ddb87341532984a1d6ca76e708db97ad Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:03:23 +0530 Subject: [PATCH 18/31] feat: Improve error handling and record appending in IcebergService - Add `AppendAlreadyDoneException` to handle specific scenario when append operation is already done - Enhance record appending process in `IcebergService` by introducing better error handling and more precise control flow --- .../jvm/DefaultExceptionHandlerProvider.java | 2 +- .../AppendAlreadyDoneException.java | 11 +++ .../jvm/iceberg/service/IcebergService.java | 82 +++++++++++-------- 3 files changed, 58 insertions(+), 37 deletions(-) create mode 100644 flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/exceptions/AppendAlreadyDoneException.java diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java index e41b2103a7..c332c2d18a 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/DefaultExceptionHandlerProvider.java @@ -26,7 +26,7 @@ public ExceptionHandler createHandler(ServerCall.List @Override public Throwable transform(Throwable t) { invoked = true; - Log.errorf(t, "Received error in gRPC call %s", t.getMessage()); + Log.errorf(t, "Received error in gRPC call: '%s'", t.getMessage()); return toStatusException(t); } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/exceptions/AppendAlreadyDoneException.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/exceptions/AppendAlreadyDoneException.java new file mode 100644 index 0000000000..0ee5fc6e67 --- /dev/null +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/exceptions/AppendAlreadyDoneException.java @@ -0,0 +1,11 @@ +package io.peerdb.flow.jvm.iceberg.exceptions; + +public class AppendAlreadyDoneException extends RuntimeException { + public AppendAlreadyDoneException() { + super(); + } + + public AppendAlreadyDoneException(String message) { + super(message); + } +} diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index 8d1a5abc9a..d6fb2355e3 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -6,6 +6,7 @@ import io.peerdb.flow.jvm.grpc.*; import io.peerdb.flow.jvm.iceberg.avro.AvroIcebergRecordConverter; import io.peerdb.flow.jvm.iceberg.catalog.CatalogLoader; +import io.peerdb.flow.jvm.iceberg.exceptions.AppendAlreadyDoneException; import io.peerdb.flow.jvm.iceberg.lock.LockManager; import io.peerdb.flow.jvm.iceberg.writer.AppendFilesWriter; import io.peerdb.flow.jvm.iceberg.writer.RecordWriterFactory; @@ -154,53 +155,62 @@ record AppendRecordTableContext(AppendRecordTableHeader tableHeader, Catalog cat AppendFilesWriter appendFilesWriter) { } - class OperationDoneException extends RuntimeException { - } public Uni appendRecordsAsync(Multi request) { // TODO the streaming code needs better error handling var tableContext = new AtomicReference(); var counter = new AtomicInteger(); - Multi appendRecordStream = request.map(record -> Pair.of(counter.getAndIncrement(), record)).map(Unchecked.function(pair -> { + var resultUni = request.map(record -> Pair.of(counter.getAndIncrement(), record)) + // Initialize the Iceberg Table and Create the AppendFilesWriter for Appending Records + .map(Unchecked.function(pair -> { + var index = pair.getKey(); var record = pair.getValue(); - if (record.getCommandCase() == AppendRecordsStreamRequest.CommandCase.TABLE_HEADER) { - var tableHeader = record.getTableHeader(); - var avroSchema = tableHeader.getSchema(); - var icebergCatalog = catalogLoader.loadCatalog(tableHeader.getTableInfo().getIcebergCatalog()); - var table = icebergCatalog.loadTable(getTableIdentifier(tableHeader.getTableInfo())); - if (isAppendAlreadyDone(table, Optional.ofNullable(tableHeader.hasIdempotencyKey() ? tableHeader.getIdempotencyKey() : null))) { - throw new OperationDoneException(); + if (index == 0) { + if (record.getCommandCase() == AppendRecordsStreamRequest.CommandCase.TABLE_HEADER) { + var tableHeader = record.getTableHeader(); + var avroSchema = tableHeader.getSchema(); + var icebergCatalog = catalogLoader.loadCatalog(tableHeader.getTableInfo().getIcebergCatalog()); + var table = icebergCatalog.loadTable(getTableIdentifier(tableHeader.getTableInfo())); + if (isAppendAlreadyDone(table, Optional.ofNullable(tableHeader.hasIdempotencyKey() ? tableHeader.getIdempotencyKey() : null))) { + throw new AppendAlreadyDoneException(String.format("Append already done for table %s with idempotency key %s", table.name(), tableHeader.getIdempotencyKey())); + } + tableContext.set(new AppendRecordTableContext(tableHeader, icebergCatalog, table, new AppendFilesWriter(recordWriterFactory, avroSchema, table))); + return Optional.empty(); } - tableContext.set(new AppendRecordTableContext(tableHeader, icebergCatalog, table, new AppendFilesWriter(recordWriterFactory, avroSchema, table))); - return Optional.empty(); + throw new IllegalArgumentException("Expected TableHeader as the first message, got " + record.getCommandCase()); } else if (record.getCommandCase() == AppendRecordsStreamRequest.CommandCase.RECORD) { return Optional.of(record.getRecord()); - } else { - throw new IllegalArgumentException("Expected TableHeader or Record"); + throw new IllegalArgumentException("Expected InsertRecord as the rest of the messages, got " + record.getCommandCase()); } - })).onFailure(throwable -> throwable instanceof OperationDoneException).recoverWithCompletion() - .filter(Optional::isPresent).map(Optional::get); - var resultUni = appendRecordStream.map(Unchecked.function(insertRecord -> { - var appendRecordTableContext = tableContext.get(); - var appendFilesWriter = appendRecordTableContext.appendFilesWriter(); - try { - appendFilesWriter.writeAvroBytesRecord(insertRecord.getRecord().toByteArray()); - } catch (IOException e) { - Log.errorf(e, "Error while converting record"); - throw new UncheckedIOException(e); - } - return true; - })).filter(obj -> !obj).collect().asList().map(Unchecked.function(ignored -> { - var appendRecordTableContext = tableContext.get(); - var tableHeader = appendRecordTableContext.tableHeader(); - var table = appendRecordTableContext.table(); - var tableInfo = tableHeader.getTableInfo(); - var idempotencyKey = Optional.ofNullable(tableHeader.hasIdempotencyKey() ? tableHeader.getIdempotencyKey() : null); - var appendFilesWriter = appendRecordTableContext.appendFilesWriter(); - var dataFiles = appendFilesWriter.complete(); - return commitDataFilesToTableWithLock(table, tableInfo, idempotencyKey, dataFiles); - })); + })).filter(Optional::isPresent).map(Optional::get) + // Write the records + .map(Unchecked.function(insertRecord -> { + var appendRecordTableContext = tableContext.get(); + var appendFilesWriter = appendRecordTableContext.appendFilesWriter(); + try { + appendFilesWriter.writeAvroBytesRecord(insertRecord.getRecord().toByteArray()); + } catch (IOException e) { + Log.errorf(e, "Error while converting record"); + throw new UncheckedIOException(e); + } + return true; + })).filter(obj -> !obj).collect().asList().replaceWithVoid() + // Now commit to the table using critical section to prevent the requirement of metadata refresh and transactions failing + .map(Unchecked.function((ignored) -> { + var appendRecordTableContext = tableContext.get(); + var tableHeader = appendRecordTableContext.tableHeader(); + var table = appendRecordTableContext.table(); + var tableInfo = tableHeader.getTableInfo(); + var idempotencyKey = Optional.ofNullable(tableHeader.hasIdempotencyKey() ? tableHeader.getIdempotencyKey() : null); + var appendFilesWriter = appendRecordTableContext.appendFilesWriter(); + var dataFiles = appendFilesWriter.complete(); + return commitDataFilesToTableWithLock(table, tableInfo, idempotencyKey, dataFiles); + })) + .onFailure(AppendAlreadyDoneException.class).recoverWithItem((error) -> { + Log.warnf( "Received AppendAlreadyCompletedException, ignoring", error.getMessage()); + return true; + }); return resultUni.map(success -> AppendRecordsStreamResponse.newBuilder().setSuccess(success).build()); } From 93830a041ce87ef5a5f3e4f19bbd7984e8d85ce9 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:25:54 +0530 Subject: [PATCH 19/31] chore: add editor config for jvm --- flow-jvm/.editorconfig | 3 +++ .../peerdb/flow/jvm/RequestLoggingInterceptor.java | 7 ++++++- .../flow/jvm/iceberg/resource/IcebergResource.java | 14 +++++++++++++- .../flow/jvm/iceberg/service/IcebergService.java | 10 +++++++++- 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 flow-jvm/.editorconfig diff --git a/flow-jvm/.editorconfig b/flow-jvm/.editorconfig new file mode 100644 index 0000000000..af4c496aec --- /dev/null +++ b/flow-jvm/.editorconfig @@ -0,0 +1,3 @@ +[*.java] +ij_java_names_count_to_use_import_on_demand = 999 +ij_java_class_count_to_use_import_on_demand = 999 diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java index 473f05efe1..361d1f5521 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java @@ -2,7 +2,12 @@ import com.google.common.base.Stopwatch; -import io.grpc.*; +import io.grpc.ForwardingServerCall; +import io.grpc.ForwardingServerCallListener; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; import io.quarkus.grpc.GlobalInterceptor; import io.quarkus.logging.Log; import jakarta.enterprise.context.ApplicationScoped; diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java index 6e43eaafd0..12b732c617 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java @@ -1,6 +1,18 @@ package io.peerdb.flow.jvm.iceberg.resource; -import io.peerdb.flow.jvm.grpc.*; +import io.peerdb.flow.jvm.grpc.AppendRecordsRequest; +import io.peerdb.flow.jvm.grpc.AppendRecordsResponse; +import io.peerdb.flow.jvm.grpc.AppendRecordsStreamRequest; +import io.peerdb.flow.jvm.grpc.AppendRecordsStreamResponse; +import io.peerdb.flow.jvm.grpc.CountRecordRequest; +import io.peerdb.flow.jvm.grpc.CountRecordResponse; +import io.peerdb.flow.jvm.grpc.CreateTableRequest; +import io.peerdb.flow.jvm.grpc.CreateTableResponse; +import io.peerdb.flow.jvm.grpc.DropTableRequest; +import io.peerdb.flow.jvm.grpc.DropTableResponse; +import io.peerdb.flow.jvm.grpc.IcebergProxyService; +import io.peerdb.flow.jvm.grpc.InsertChangesRequest; +import io.peerdb.flow.jvm.grpc.InsertChangesResponse; import io.peerdb.flow.jvm.iceberg.service.IcebergService; import io.quarkus.grpc.GrpcService; import io.smallrye.common.annotation.Blocking; diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index d6fb2355e3..3b9616f1a0 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -3,7 +3,15 @@ import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.collect.Streams; -import io.peerdb.flow.jvm.grpc.*; +import io.peerdb.flow.jvm.grpc.AppendRecordTableHeader; +import io.peerdb.flow.jvm.grpc.AppendRecordsRequest; +import io.peerdb.flow.jvm.grpc.AppendRecordsStreamRequest; +import io.peerdb.flow.jvm.grpc.AppendRecordsStreamResponse; +import io.peerdb.flow.jvm.grpc.BranchOptions; +import io.peerdb.flow.jvm.grpc.CountRecordRequest; +import io.peerdb.flow.jvm.grpc.InsertRecord; +import io.peerdb.flow.jvm.grpc.RecordChange; +import io.peerdb.flow.jvm.grpc.TableInfo; import io.peerdb.flow.jvm.iceberg.avro.AvroIcebergRecordConverter; import io.peerdb.flow.jvm.iceberg.catalog.CatalogLoader; import io.peerdb.flow.jvm.iceberg.exceptions.AppendAlreadyDoneException; From 5bbb18e63eec172d867abc2ab9b42e2ebe53c4f0 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:45:39 +0530 Subject: [PATCH 20/31] chore: try switching gh runner to fix build --- .github/workflows/flow-jvm-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flow-jvm-build.yml b/.github/workflows/flow-jvm-build.yml index 4e8372ee74..de6175baf1 100644 --- a/.github/workflows/flow-jvm-build.yml +++ b/.github/workflows/flow-jvm-build.yml @@ -12,7 +12,7 @@ jobs: name: Build & Test Flow JVM strategy: matrix: - runner: [ubicloud-standard-2-ubuntu-2204-arm] + runner: [ubuntu-latest] runs-on: ${{ matrix.runner }} steps: - name: checkout From 237ac321e4e8c63c792aba4eee38b894fd0ef934 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:13:40 +0530 Subject: [PATCH 21/31] chore: lint fix --- flow/connectors/iceberg/qrep.go | 3 +-- flow/peerdbenv/features/iceberg.go | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go index 4da8c44c55..af64c4d241 100644 --- a/flow/connectors/iceberg/qrep.go +++ b/flow/connectors/iceberg/qrep.go @@ -3,7 +3,6 @@ package iceberg import ( "context" "fmt" - "github.com/PeerDB-io/peer-flow/peerdbenv/features" "log/slog" "time" @@ -13,6 +12,7 @@ import ( "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/peerdbenv/features" ) func (c *IcebergConnector) SyncQRepRecords( @@ -27,7 +27,6 @@ func (c *IcebergConnector) SyncQRepRecords( return c.sendRecordsJoined(ctx, config, partition, stream) } -//nolint:unused func (c *IcebergConnector) sendRecordsJoined( ctx context.Context, config *protos.QRepConfig, diff --git a/flow/peerdbenv/features/iceberg.go b/flow/peerdbenv/features/iceberg.go index 3fb839f738..e079983744 100644 --- a/flow/peerdbenv/features/iceberg.go +++ b/flow/peerdbenv/features/iceberg.go @@ -2,10 +2,11 @@ package features import ( "context" - "github.com/PeerDB-io/peer-flow/logger" - "github.com/PeerDB-io/peer-flow/peerdbenv" "log/slog" "strconv" + + "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/peerdbenv" ) func IcebergFeatureStreamingEnabled(ctx context.Context) bool { From a6c83db7c266a3fc2eca88c18417810b3baf0f1f Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 02:57:44 +0530 Subject: [PATCH 22/31] feat(LockManager): change scope from Dependent to ApplicationScoped --- .../java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java index bae324d20d..e5e4f95e32 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/lock/LockManager.java @@ -2,11 +2,11 @@ import com.google.common.util.concurrent.Striped; -import jakarta.enterprise.context.Dependent; +import jakarta.enterprise.context.ApplicationScoped; import java.util.concurrent.locks.Lock; -@Dependent +@ApplicationScoped public class LockManager { int stripeCount = 10_000; From 79129a9ff5d802390522af2431b498467c72ee10 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 03:35:02 +0530 Subject: [PATCH 23/31] feat: enable streaming and nit fixes --- .../io/peerdb/flow/jvm/iceberg/service/IcebergService.java | 1 - flow/connectors/iceberg/iceberg.go | 3 +-- flow/connectors/iceberg/qrep.go | 1 - flow/peerdbenv/features/iceberg.go | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index 3b9616f1a0..c929ee3ed7 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -165,7 +165,6 @@ record AppendRecordTableContext(AppendRecordTableHeader tableHeader, Catalog cat public Uni appendRecordsAsync(Multi request) { - // TODO the streaming code needs better error handling var tableContext = new AtomicReference(); var counter = new AtomicInteger(); var resultUni = request.map(record -> Pair.of(counter.getAndIncrement(), record)) diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go index 34a397212f..baafdcb165 100644 --- a/flow/connectors/iceberg/iceberg.go +++ b/flow/connectors/iceberg/iceberg.go @@ -137,7 +137,6 @@ func (c *IcebergConnector) SetupNormalizedTable( softDeleteColName string, syncedAtColName string, ) (bool, error) { - // TODO add soft delete column in the schema qFields := make([]qvalue.QField, len(tableSchema.Columns)) for i, fieldDescription := range tableSchema.Columns { colName := fieldDescription.Name @@ -165,7 +164,7 @@ func (c *IcebergConnector) SetupNormalizedTable( } // TODO save to a buffer and call when Finish is called - // TODO maybe can later migrate to a streaming rpc with transaction support + // TODO maybe later migrate to a streaming rpc with transaction support tableResponse, err := c.proxyClient.CreateTable(ctx, &protos.CreateTableRequest{ TableInfo: &protos.TableInfo{ Namespace: nil, diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go index af64c4d241..38edbd8732 100644 --- a/flow/connectors/iceberg/qrep.go +++ b/flow/connectors/iceberg/qrep.go @@ -235,6 +235,5 @@ func (c *IcebergConnector) SetupQRepMetadataTables(_ context.Context, config *pr func (c *IcebergConnector) IsQRepPartitionSynced(ctx context.Context, config *protos.IsQRepPartitionSyncedInput, ) (bool, error) { - // TODO look at this return c.PostgresMetadata.IsQRepPartitionSynced(ctx, config) } diff --git a/flow/peerdbenv/features/iceberg.go b/flow/peerdbenv/features/iceberg.go index e079983744..0a38f5ff9b 100644 --- a/flow/peerdbenv/features/iceberg.go +++ b/flow/peerdbenv/features/iceberg.go @@ -10,7 +10,7 @@ import ( ) func IcebergFeatureStreamingEnabled(ctx context.Context) bool { - strValue := peerdbenv.GetEnvString("ICEBERG_FEATURE_STREAMING_ENABLED", "false") + strValue := peerdbenv.GetEnvString("ICEBERG_FEATURE_STREAMING_ENABLED", "true") value, err := strconv.ParseBool(strValue) if err != nil { // only log and return false From 55a19abe04d57481441b04644094a65bdefc8452 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 03:57:30 +0530 Subject: [PATCH 24/31] fix: UI showing Iceberg as unrecognized --- ui/components/PeerTypeComponent.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui/components/PeerTypeComponent.tsx b/ui/components/PeerTypeComponent.tsx index fbcc565e38..aa13a8f566 100644 --- a/ui/components/PeerTypeComponent.tsx +++ b/ui/components/PeerTypeComponent.tsx @@ -28,6 +28,8 @@ export const DBTypeToGoodText = (ptype?: DBType) => { return 'PubSub'; case DBType.ELASTICSEARCH: return 'Elasticsearch'; + case DBType.ICEBERG: + return 'Iceberg'; default: return 'Unrecognised'; } From 2dff0bc0a586557b2200bf62cab71e883f920386 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 04:05:54 +0530 Subject: [PATCH 25/31] fix: setup for iceberg --- flow/connectors/core.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flow/connectors/core.go b/flow/connectors/core.go index 02a8178b03..3b5ddcbbef 100644 --- a/flow/connectors/core.go +++ b/flow/connectors/core.go @@ -348,6 +348,12 @@ func LoadPeer(ctx context.Context, catalogPool *pgxpool.Pool, peerName string) ( return nil, fmt.Errorf("failed to unmarshal Elasticsearch config: %w", err) } peer.Config = &protos.Peer_ElasticsearchConfig{ElasticsearchConfig: &config} + case protos.DBType_ICEBERG: + var config protos.IcebergConfig + if err := proto.Unmarshal(peerOptions, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal Iceberg config: %w", err) + } + peer.Config = &protos.Peer_IcebergConfig{IcebergConfig: &config} default: return nil, fmt.Errorf("unsupported peer type: %s", peer.Type) } From 6ea513764a73397d6244b953c1c2686163d88fc5 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 04:59:11 +0530 Subject: [PATCH 26/31] chore: dependency updates --- flow-jvm/build.gradle | 1 - flow-jvm/gradle.properties | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/flow-jvm/build.gradle b/flow-jvm/build.gradle index a741a4a9c7..ae782c656f 100644 --- a/flow-jvm/build.gradle +++ b/flow-jvm/build.gradle @@ -109,5 +109,4 @@ compileTestJava { quarkus { quarkusBuildProperties.put("quarkus.grpc.codegen.proto-directory", "${project.projectDir}/../protos") - quarkusBuildProperties.put("quarkus.grpc.codegen.exclude-filter", ".*") } diff --git a/flow-jvm/gradle.properties b/flow-jvm/gradle.properties index d87a1b7a1f..0225dfbc19 100644 --- a/flow-jvm/gradle.properties +++ b/flow-jvm/gradle.properties @@ -2,6 +2,6 @@ #Fri May 24 04:59:17 IST 2024 quarkusPlatformArtifactId=quarkus-bom quarkusPlatformGroupId=io.quarkus.platform -quarkusPlatformVersion=3.11.1 +quarkusPlatformVersion=3.11.3 quarkusPluginId=io.quarkus -quarkusPluginVersion=3.11.1 +quarkusPluginVersion=3.11.3 From 78a5468950d7d43a0d34df81d2e303cdf4a0857e Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 05:06:43 +0530 Subject: [PATCH 27/31] chore: nit fixes --- protos/peers.proto | 2 -- ui/components/PeerForms/Iceberg/IcebergConfig.tsx | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/protos/peers.proto b/protos/peers.proto index 28d551acc3..8454049986 100644 --- a/protos/peers.proto +++ b/protos/peers.proto @@ -197,8 +197,6 @@ message CommonIcebergCatalog { optional int32 client_pool_size = 4; optional bool cache_enabled = 5; map hadoop_properties = 6; - // This helps in testing, where we can pass additional properties to the catalog - // map additional_properties = 6; } message HiveIcebergCatalog {} diff --git a/ui/components/PeerForms/Iceberg/IcebergConfig.tsx b/ui/components/PeerForms/Iceberg/IcebergConfig.tsx index 55ade7b5fa..49aeef44b6 100644 --- a/ui/components/PeerForms/Iceberg/IcebergConfig.tsx +++ b/ui/components/PeerForms/Iceberg/IcebergConfig.tsx @@ -162,7 +162,7 @@ const IcebergConfigForm = ({ icebergConfig, setter }: IcebergConfigProps) => { - Choose specific catalog + Choose Iceberg Catalog Type Date: Tue, 25 Jun 2024 05:08:31 +0530 Subject: [PATCH 28/31] chore: update quarkus cli version in sdkmanrc --- .sdkmanrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sdkmanrc b/.sdkmanrc index 00e6432dd9..64fb7e6bdc 100644 --- a/.sdkmanrc +++ b/.sdkmanrc @@ -1,5 +1,5 @@ # Enable auto-env through the sdkman_auto_env config # Add key=value pairs of SDKs to use below java=21.0.3-tem -quarkus=3.11.1 +quarkus=3.11.3 gradle=8.6 From 0c5d016c8e7884fbc2c0ce909c1594d4ecfbe9c1 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 05:35:02 +0530 Subject: [PATCH 29/31] chore: cleaning up code --- .../flow/jvm/RequestLoggingInterceptor.java | 1 - .../jvm/iceberg/resource/IcebergResource.java | 12 +++++++ .../jvm/iceberg/service/IcebergService.java | 35 +++---------------- flow/connectors/core.go | 10 +++--- flow/connectors/iceberg/qrep.go | 3 +- flow/peerdbenv/features/iceberg.go | 6 ++-- 6 files changed, 25 insertions(+), 42 deletions(-) diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java index 361d1f5521..ed18b809e7 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/RequestLoggingInterceptor.java @@ -40,7 +40,6 @@ public CallListener(ServerCallHandler serverCallHandler, ServerCall @Override public void onMessage(ReqT message) { -// Log.debugf("Received request for method: %s: {%s}", serverCall.getMethodDescriptor().getFullMethodName(), message); super.onMessage(message); } diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java index 12b732c617..a47c2d876f 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/resource/IcebergResource.java @@ -72,8 +72,15 @@ public Uni insertChanges(InsertChangesRequest request) { } + /** + * Append records to the iceberg table with records all encoded in the request body + * @deprecated Use {@link #streamingAppendRecords(Multi)} instead for better performance + * @param request AppendRecordsRequest containing the records to be appended along with table info + * @return AppendRecordsResponse containing the success status of the operation + */ @Blocking @Override + @Deprecated public Uni appendRecords(AppendRecordsRequest request) { return Uni.createFrom().item(() -> AppendRecordsResponse.newBuilder() .setSuccess( @@ -81,6 +88,11 @@ public Uni appendRecords(AppendRecordsRequest request) { .build()); } + /** + * Append records to the iceberg table with records streamed in the request body + * @param request AppendRecordsStreamRequest containing the records to be appended along with table info as the first record + * @return AppendRecordsStreamResponse containing the success status of the operation + */ @Blocking @Override public Uni streamingAppendRecords(Multi request) { diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index c929ee3ed7..5131458ff9 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -127,36 +127,9 @@ public boolean dropTable(TableInfo tableInfo, boolean purge) { } public boolean processAppendRecordsRequest(AppendRecordsRequest request) { - return appendRecords(request.getTableHeader().getTableInfo(), - request.getTableHeader().getSchema(), - request.getRecordsList(), - Optional.ofNullable(request.getTableHeader().hasIdempotencyKey() ? request.getTableHeader().getIdempotencyKey() : null)); - } - - public boolean processAppendRecordsStreamRequest(Multi request) { - Log.info("Received a Process Append Records Stream Request, will wait for first message"); - // Lets error out after 1 minute - var firstMessage = Uni.createFrom().multi(request).await().atMost(Duration.ofSeconds(30)); - if (firstMessage.getCommandCase() != AppendRecordsStreamRequest.CommandCase.TABLE_HEADER && !firstMessage.hasTableHeader()) { - Log.errorf("TableHeader should be present in the first message, found %s", firstMessage.getCommandCase()); - throw new IllegalArgumentException("TableHeader should be present in the first message"); - } - var tableHeader = firstMessage.getTableHeader(); - var tableInfo = tableHeader.getTableInfo(); - var avroSchema = tableHeader.getSchema(); - Log.infof("Received a Process Append Records Stream Request for table %s, will publish stream for appending records", tableInfo.getTableName()); - var insertRecordStream = request.map(Unchecked.function(message -> { - if (message.hasRecord()) { - return message.getRecord(); - } else { - throw new IllegalArgumentException("Only InsertRecord is supported"); - } - })).subscribe().asStream(); - return appendRecords(tableInfo, avroSchema, insertRecordStream, Optional.ofNullable(tableHeader.hasIdempotencyKey() ? tableHeader.getIdempotencyKey() : null)); - } - - private boolean appendRecords(TableInfo tableInfo, String avroSchema, List insertRecords, Optional idempotencyKey) { - return appendRecords(tableInfo, avroSchema, insertRecords.stream(), idempotencyKey); + String avroSchema = request.getTableHeader().getSchema(); + Optional idempotencyKey = Optional.ofNullable(request.getTableHeader().hasIdempotencyKey() ? request.getTableHeader().getIdempotencyKey() : null); + return appendRecords(request.getTableHeader().getTableInfo(), avroSchema, request.getRecordsList().stream(), idempotencyKey); } record AppendRecordTableContext(AppendRecordTableHeader tableHeader, Catalog catalog, Table table, @@ -215,7 +188,7 @@ var record = pair.getValue(); return commitDataFilesToTableWithLock(table, tableInfo, idempotencyKey, dataFiles); })) .onFailure(AppendAlreadyDoneException.class).recoverWithItem((error) -> { - Log.warnf( "Received AppendAlreadyCompletedException, ignoring", error.getMessage()); + Log.warnf("Received AppendAlreadyCompletedException, ignoring", error.getMessage()); return true; }); return resultUni.map(success -> AppendRecordsStreamResponse.newBuilder().setSuccess(success).build()); diff --git a/flow/connectors/core.go b/flow/connectors/core.go index 3b5ddcbbef..1d290878ba 100644 --- a/flow/connectors/core.go +++ b/flow/connectors/core.go @@ -14,7 +14,7 @@ import ( connclickhouse "github.com/PeerDB-io/peer-flow/connectors/clickhouse" connelasticsearch "github.com/PeerDB-io/peer-flow/connectors/connelasticsearch" conneventhub "github.com/PeerDB-io/peer-flow/connectors/eventhub" - "github.com/PeerDB-io/peer-flow/connectors/iceberg" + conniceberg "github.com/PeerDB-io/peer-flow/connectors/iceberg" connkafka "github.com/PeerDB-io/peer-flow/connectors/kafka" connmysql "github.com/PeerDB-io/peer-flow/connectors/mysql" connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" @@ -386,7 +386,7 @@ func GetConnector(ctx context.Context, config *protos.Peer) (Connector, error) { case *protos.Peer_ElasticsearchConfig: return connelasticsearch.NewElasticsearchConnector(ctx, inner.ElasticsearchConfig) case *protos.Peer_IcebergConfig: - return iceberg.NewIcebergConnector(ctx, inner.IcebergConfig) + return conniceberg.NewIcebergConnector(ctx, inner.IcebergConfig) default: return nil, errors.ErrUnsupported } @@ -437,7 +437,7 @@ var ( _ CDCSyncConnector = &conns3.S3Connector{} _ CDCSyncConnector = &connclickhouse.ClickhouseConnector{} _ CDCSyncConnector = &connelasticsearch.ElasticsearchConnector{} - _ CDCSyncConnector = &iceberg.IcebergConnector{} + _ CDCSyncConnector = &conniceberg.IcebergConnector{} _ CDCSyncPgConnector = &connpostgres.PostgresConnector{} @@ -469,7 +469,7 @@ var ( _ QRepSyncConnector = &conns3.S3Connector{} _ QRepSyncConnector = &connclickhouse.ClickhouseConnector{} _ QRepSyncConnector = &connelasticsearch.ElasticsearchConnector{} - _ QRepSyncConnector = &iceberg.IcebergConnector{} + _ QRepSyncConnector = &conniceberg.IcebergConnector{} _ QRepSyncPgConnector = &connpostgres.PostgresConnector{} @@ -484,7 +484,7 @@ var ( _ ValidationConnector = &connclickhouse.ClickhouseConnector{} _ ValidationConnector = &connbigquery.BigQueryConnector{} _ ValidationConnector = &conns3.S3Connector{} - _ ValidationConnector = &iceberg.IcebergConnector{} + _ ValidationConnector = &conniceberg.IcebergConnector{} _ Connector = &connmysql.MySqlConnector{} ) diff --git a/flow/connectors/iceberg/qrep.go b/flow/connectors/iceberg/qrep.go index 38edbd8732..6ce51f9ec5 100644 --- a/flow/connectors/iceberg/qrep.go +++ b/flow/connectors/iceberg/qrep.go @@ -21,7 +21,7 @@ func (c *IcebergConnector) SyncQRepRecords( partition *protos.QRepPartition, stream *model.QRecordStream, ) (int, error) { - if features.IcebergFeatureStreamingEnabled(ctx) { + if !features.IcebergFeatureStreamingDisabled(ctx) { return c.streamRecords(ctx, config, partition, stream) } return c.sendRecordsJoined(ctx, config, partition, stream) @@ -74,7 +74,6 @@ func (c *IcebergConnector) sendRecordsJoined( if err != nil { return 0, fmt.Errorf("failed to convert Avro map to binary: %w", err) } - // TODO remove this log binaryRecords = append(binaryRecords, &protos.InsertRecord{ Record: native, diff --git a/flow/peerdbenv/features/iceberg.go b/flow/peerdbenv/features/iceberg.go index 0a38f5ff9b..4ece9c001e 100644 --- a/flow/peerdbenv/features/iceberg.go +++ b/flow/peerdbenv/features/iceberg.go @@ -9,12 +9,12 @@ import ( "github.com/PeerDB-io/peer-flow/peerdbenv" ) -func IcebergFeatureStreamingEnabled(ctx context.Context) bool { - strValue := peerdbenv.GetEnvString("ICEBERG_FEATURE_STREAMING_ENABLED", "true") +func IcebergFeatureStreamingDisabled(ctx context.Context) bool { + strValue := peerdbenv.GetEnvString("ICEBERG_FEATURE_STREAMING_DISABLED", "false") value, err := strconv.ParseBool(strValue) if err != nil { // only log and return false - logger.LoggerFromCtx(ctx).Error("Failed to get ICEBERG_FEATURE_STREAMING_ENABLED", slog.Any("error", err)) + logger.LoggerFromCtx(ctx).Error("Failed to get ICEBERG_FEATURE_STREAMING_DISABLED", slog.Any("error", err)) return false } return value From 74bca1b011df65424dc101ee73a7b25d81fe85a6 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 06:28:21 +0530 Subject: [PATCH 30/31] feat: identifier fields support --- .../flow/jvm/iceberg/service/IcebergService.java | 14 ++++++++------ flow/connectors/iceberg/iceberg.go | 12 ++++++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java index 5131458ff9..8e2f32e697 100644 --- a/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java +++ b/flow-jvm/src/main/java/io/peerdb/flow/jvm/iceberg/service/IcebergService.java @@ -44,10 +44,12 @@ import java.time.Duration; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import java.util.stream.Stream; @ApplicationScoped @@ -94,12 +96,12 @@ public Table createTable(TableInfo tableInfo, String schema) { var catalog = catalogLoader.loadCatalog(icebergCatalog); var typeSchema = getIcebergSchema(schema); // TODO Below require that the primary keys are non-null -// var fieldList = typeSchema.columns(); -// var primaryKeyFieldIds = request.getTableInfo().getPrimaryKeyList().stream().map(pk -> -// Objects.requireNonNull(typeSchema.findField(pk), String.format("Primary key %s not found in schema", pk)).fieldId() -// ).collect(Collectors.toSet()); -// var icebergSchema = new Schema(fieldList, primaryKeyFieldIds); - var icebergSchema = typeSchema; + var fieldList = typeSchema.columns(); + var primaryKeyFieldIds = tableInfo.getPrimaryKeyList().stream().map(pk -> + Objects.requireNonNull(typeSchema.findField(pk), String.format("Primary key %s not found in schema", pk)).fieldId() + ).collect(Collectors.toSet()); + var icebergSchema = new Schema(fieldList, primaryKeyFieldIds); + Preconditions.checkArgument(icebergSchema.asStruct().equals(typeSchema.asStruct()), "Primary key based schema not equivalent to type schema [%s!=%s]", icebergSchema.asStruct(), typeSchema.asStruct()); var tableIdentifier = getTableIdentifier(tableInfo); diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go index baafdcb165..85f57240d1 100644 --- a/flow/connectors/iceberg/iceberg.go +++ b/flow/connectors/iceberg/iceberg.go @@ -137,6 +137,12 @@ func (c *IcebergConnector) SetupNormalizedTable( softDeleteColName string, syncedAtColName string, ) (bool, error) { + + primaryKeyColumns := make(map[string]struct{}, len(tableSchema.PrimaryKeyColumns)) + for _, col := range tableSchema.PrimaryKeyColumns { + primaryKeyColumns[col] = struct{}{} + } + qFields := make([]qvalue.QField, len(tableSchema.Columns)) for i, fieldDescription := range tableSchema.Columns { colName := fieldDescription.Name @@ -145,13 +151,15 @@ func (c *IcebergConnector) SetupNormalizedTable( if qValueKind == qvalue.QValueKindNumeric { precision, scale = datatypes.ParseNumericTypmod(fieldDescription.TypeModifier) } + + _, isPrimaryKey := primaryKeyColumns[colName] + qField := qvalue.QField{ Name: colName, Type: qValueKind, Precision: precision, Scale: scale, - // TODO check this - Nullable: true, + Nullable: !isPrimaryKey, } qFields[i] = qField } From cb190cbec457c9cf0ac34452cebaa5fd94b2eea6 Mon Sep 17 00:00:00 2001 From: Kunal Gupta <39487888+iamKunalGupta@users.noreply.github.com> Date: Tue, 25 Jun 2024 07:54:03 +0530 Subject: [PATCH 31/31] chore: lint fix --- flow/connectors/iceberg/iceberg.go | 1 - 1 file changed, 1 deletion(-) diff --git a/flow/connectors/iceberg/iceberg.go b/flow/connectors/iceberg/iceberg.go index 85f57240d1..9f03d2f145 100644 --- a/flow/connectors/iceberg/iceberg.go +++ b/flow/connectors/iceberg/iceberg.go @@ -137,7 +137,6 @@ func (c *IcebergConnector) SetupNormalizedTable( softDeleteColName string, syncedAtColName string, ) (bool, error) { - primaryKeyColumns := make(map[string]struct{}, len(tableSchema.PrimaryKeyColumns)) for _, col := range tableSchema.PrimaryKeyColumns { primaryKeyColumns[col] = struct{}{}