Merge branch 'main' into es-8.2

jaegertracing · Mar 6, 2024 · d7331eb · d7331eb
2 parents c81c12b + e6055bf
commit d7331eb
Show file tree

Hide file tree

Showing 10 changed files with 170 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -88,6 +88,15 @@ Example usage:
 $ STORAGE=elasticsearch ES_NODES=http://localhost:9200 java -jar jaeger-spark-dependencies.jar
 ```
 
+## Design
+
+At a high-level, this job does the following:
+  * read lots of spans from a time period
+  * group them by traceId
+  * construct a graph using parent-child relationships expressed in span references
+  * for each edge `(parent span, child span)` output `(parent service, child service, count)`
+  * write the results to the database (e.g. `dependencies_v2` table in [Cassandra](https://github.com/jaegertracing/jaeger/blob/12e44faabf10cdd866391b78933eec5d6ac50fa9/plugin/storage/cassandra/schema/v004.cql.tmpl#L186))
+
 ## Building locally
 To build the job locally and run tests:
 ```bash

diff --git a/jaeger-spark-dependencies-cassandra/pom.xml b/jaeger-spark-dependencies-cassandra/pom.xml
@@ -37,13 +37,30 @@
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${version.scala.binary}</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-annotations</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
       <groupId>com.datastax.spark</groupId>
       <artifactId>spark-cassandra-connector-unshaded_${version.scala.binary}</artifactId>
       <version>${spark-cassandra-connector.version}</version>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>1.26.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-xml</artifactId>
+      <version>10.0.16</version>
+    </dependency>
 
     <dependency>
       <groupId>${project.groupId}</groupId>
@@ -55,5 +72,17 @@
       <artifactId>testcontainers</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.codahale.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>3.0.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-lang</groupId>
+      <artifactId>commons-lang</artifactId>
+      <version>2.6</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>
diff --git a/...cies-common/src/main/java/io/jaegertracing/spark/dependencies/SpansToDependencyLinks.java b/...cies-common/src/main/java/io/jaegertracing/spark/dependencies/SpansToDependencyLinks.java
@@ -26,12 +26,12 @@
 import java.util.Optional;
 import java.util.Set;
 
-import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.FlatMapFunction;
 
 /**
  * @author Pavol Loffay
  */
-public class SpansToDependencyLinks implements Function<Iterable<Span>, Iterable<Dependency>>{
+public class SpansToDependencyLinks implements FlatMapFunction<Iterable<Span>, Dependency>{
 
     /**
      * Derives dependency links based on supplied spans.
@@ -48,7 +48,7 @@ public SpansToDependencyLinks(String peerServiceTag){
     }
 
     @Override
-    public Iterable<Dependency> call(Iterable<Span> trace) {
+    public java.util.Iterator<Dependency> call(Iterable<Span> trace) {
         Map<Long, Set<Span>> spanMap = new LinkedHashMap<>();
         Map<Long, Set<Span>> spanChildrenMap = new LinkedHashMap<>();
         for (Span span: trace) {
@@ -111,7 +111,7 @@ public Iterable<Dependency> call(Iterable<Span> trace) {
               }
             }
         }
-        return result;
+        return result.iterator();
     }
 
     static Optional<Span> serverSpan(Set<Span> sharedSpans) {

diff --git a/jaeger-spark-dependencies-elasticsearch/pom.xml b/jaeger-spark-dependencies-elasticsearch/pom.xml
@@ -30,6 +30,12 @@
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>jaeger-spark-dependencies-common</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-annotations</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>

diff --git a/...st/java/io/jaegertracing/spark/dependencies/elastic/ElasticsearchDependenciesJobTest.java b/...st/java/io/jaegertracing/spark/dependencies/elastic/ElasticsearchDependenciesJobTest.java
@@ -24,6 +24,7 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import org.apache.thrift.transport.TTransportException;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -48,7 +49,7 @@ public static void beforeClass() {
   }
 
   @Before
-  public void before() {
+  public void before() throws TTransportException {
     JaegerTracer initStorageTracer = TracersGenerator.createJaeger(UUID.randomUUID().toString(), collectorUrl).getA();
     initStorageTracer.buildSpan(UUID.randomUUID().toString()).withTag("foo", "bar").start().finish();
     initStorageTracer.close();

diff --git a/...test/java/io/jaegertracing/spark/dependencies/elastic/JaegerElasticsearchEnvironment.java b/...test/java/io/jaegertracing/spark/dependencies/elastic/JaegerElasticsearchEnvironment.java
@@ -48,7 +48,7 @@ public class JaegerElasticsearchEnvironment {
 
   public static String elasticsearchVersion() {
     String version = System.getProperty("elasticsearch.version", System.getenv("ELASTICSEARCH_VERSION"));
-    return version != null ? version : "5.6.9";
+    return version != null ? version : "7.17.10";
   }
 
   public void start(Map<String, String> jaegerEnvs, String jaegerVersion, String elasticsearchVersion) {

diff --git a/...dencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/TracersGenerator.java b/...dencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/TracersGenerator.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import org.apache.thrift.transport.TTransportException;
 import zipkin.Span;
 import zipkin.reporter.AsyncReporter;
 import zipkin.reporter.Encoding;
@@ -89,7 +90,7 @@ public Flushable flushable() {
     }
   }
 
-  public static List<TracerHolder<JaegerTracer>> generateJaeger(int number, String collectorUrl) {
+  public static List<TracerHolder<JaegerTracer>> generateJaeger(int number, String collectorUrl) throws TTransportException {
     List<TracerHolder<JaegerTracer>> tracers = new ArrayList<>(number);
     for (int i = 0; i < number; i++) {
       String serviceName = serviceName();
@@ -99,7 +100,7 @@ public static List<TracerHolder<JaegerTracer>> generateJaeger(int number, String
     return tracers;
   }
 
-  public static Tuple<JaegerTracer, Flushable> createJaeger(String serviceName, String collectorUrl) {
+  public static Tuple<JaegerTracer, Flushable> createJaeger(String serviceName, String collectorUrl) throws TTransportException {
     HttpSender sender = new HttpSender.Builder(collectorUrl + "/api/traces").build();
     RemoteReporter reporter = new RemoteReporter.Builder()
         .withSender(sender)

diff --git a/...s-test/src/test/java/io/jaegertracing/spark/dependencies/test/tree/TreeGeneratorTest.java b/...s-test/src/test/java/io/jaegertracing/spark/dependencies/test/tree/TreeGeneratorTest.java
@@ -22,6 +22,7 @@
 import io.jaegertracing.spark.dependencies.test.tree.TracingWrapper.ZipkinWrapper;
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.thrift.transport.TTransportException;
 import org.junit.Test;
 
 /**
@@ -30,7 +31,7 @@
 public class TreeGeneratorTest {
 
   @Test
-  public void testGenerateOne() {
+  public void testGenerateOne() throws TTransportException {
     Node<JaegerWrapper> root = new TreeGenerator(TracersGenerator.generateJaeger(1, "http://localhost"))
         .generateTree(1, 3);
     assertEquals(0, root.getDescendants().size());
@@ -40,7 +41,7 @@ public void testGenerateOne() {
   }
 
   @Test
-  public void testBranchingFactorOne() {
+  public void testBranchingFactorOne() throws TTransportException {
     Node<JaegerWrapper> root = new TreeGenerator(TracersGenerator.generateJaeger(1, "http://localhost"))
         .generateTree(16, 3);
     List<Node> nodes = new ArrayList<>();

diff --git a/jaeger-spark-dependencies/pom.xml b/jaeger-spark-dependencies/pom.xml
@@ -69,6 +69,12 @@
                 </filter>
                 <filter>
                   <artifact>log4j:log4j</artifact>
+                  <includes>
+                    <include>org/apache/log4j/spi/LoggingEvent.class</include>
+                  </includes>
+                </filter>
+                <filter>
+                  <artifact>org.apache.logging.log4j:log4j-*</artifact>
                   <includes>
                     <include>**</include>
                   </includes>

diff --git a/pom.xml b/pom.xml
@@ -37,23 +37,25 @@
     <project.build.resourceEncoding>UTF-8</project.build.resourceEncoding>
 
     <version.scala.binary>2.12</version.scala.binary>
-    <version.org.apache.spark>2.4.8</version.org.apache.spark>
-    <version.io.jaegertracing>0.34.0</version.io.jaegertracing>
+    <version.org.apache.spark>3.5.1</version.org.apache.spark>
+    <version.io.jaegertracing>0.34.3</version.io.jaegertracing>
     <version.io.opentracing>0.31.0</version.io.opentracing>
-    <version.io.zipkin.brave-brave>4.6.0</version.io.zipkin.brave-brave>
+    <version.io.zipkin.brave-brave>4.19.3</version.io.zipkin.brave-brave>
     <version.io.zipkin.reporter-zipkin-sender-okhttp3>1.0.2</version.io.zipkin.reporter-zipkin-sender-okhttp3>
     <version.junit>4.13.2</version.junit>
-    <version.org.assertj>3.22.0</version.org.assertj>
-    <version.org.testcontainers>1.14.3</version.org.testcontainers>
+    <version.org.assertj>3.24.2</version.org.assertj>
+    <version.org.testcontainers>1.18.1</version.org.testcontainers>
     <version.com.squareup.okhttp3-okhttp>3.14.9</version.com.squareup.okhttp3-okhttp>
-    <version.org.awaitility-awaitility>4.1.1</version.org.awaitility-awaitility>
+    <version.org.awaitility-awaitility>4.2.0</version.org.awaitility-awaitility>
 
     <version.maven-license-plugin>3.0</version.maven-license-plugin>
-    <version.maven-compiler-plugin>3.6.1</version.maven-compiler-plugin>
-    <version.maven-install-plugin>2.5.2</version.maven-install-plugin>
-    <version.maven-source-plugin>3.0.1</version.maven-source-plugin>
+    <version.maven-compiler-plugin>3.11.0</version.maven-compiler-plugin>
+    <version.maven-install-plugin>3.1.1</version.maven-install-plugin>
+    <version.maven-source-plugin>3.2.1</version.maven-source-plugin>
     <version.maven-plugin>0.3.4</version.maven-plugin>
-    <version.maven-shade-plugin>3.1.0</version.maven-shade-plugin>
+    <version.maven-shade-plugin>3.5.2</version.maven-shade-plugin>
+    <version.jackson>2.15.3</version.jackson>
+    <version.hadoop.client>3.3.6</version.hadoop.client>
   </properties>
 
   <inceptionYear>2017</inceptionYear>
@@ -107,6 +109,26 @@
             <groupId>commons-codec</groupId>
             <artifactId>commons-codec</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-scala_${version.scala.binary}</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client-api</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client-runtime</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
 
@@ -115,13 +137,17 @@
         <artifactId>testcontainers</artifactId>
         <version>${version.org.testcontainers}</version>
       </dependency>
-
       <!-- Forcibly bump Commons Collection version to avoid CVE-2015-7501 -->
       <dependency>
         <groupId>commons-collections</groupId>
         <artifactId>commons-collections</artifactId>
         <version>3.2.2</version>
       </dependency>
+      <dependency>
+        <groupId>commons-beanutils</groupId>
+        <artifactId>commons-beanutils</artifactId>
+        <version>1.9.4</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -136,8 +162,77 @@
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-sql_${version.scala.binary}</artifactId>
       <version>${version.org.apache.spark}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.avro</groupId>
+          <artifactId>avro</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.module</groupId>
+          <artifactId>jackson-module-scala_${version.scala.binary}</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
-
+    <!-- BEGIN dependecy override versions -->
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>3.25.3</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.zookeeper</groupId>
+      <artifactId>zookeeper</artifactId>
+      <version>3.7.2</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+      <version>${version.jackson}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${version.jackson}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.module</groupId>
+      <artifactId>jackson-module-scala_${version.scala.binary}</artifactId>
+      <version>${version.jackson}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.jaxrs</groupId>
+      <artifactId>jackson-jaxrs-base</artifactId>
+      <version>${version.jackson}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.jaxrs</groupId>
+      <artifactId>jackson-jaxrs-json-provider</artifactId>
+      <version>${version.jackson}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <version>${version.hadoop.client}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-protobuf_3_7</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-all</artifactId>
+      <version>4.1.100.Final</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.avro</groupId>
+      <artifactId>avro</artifactId>
+      <version>1.11.3</version>
+    </dependency>
+    <!-- END dependency override versions -->
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>