diff --git a/telemetry/build.gradle b/telemetry/build.gradle index 5961cb10260..c85d0228b5b 100644 --- a/telemetry/build.gradle +++ b/telemetry/build.gradle @@ -48,9 +48,6 @@ dependencies { testImplementation project(':utils:test-utils') testImplementation group: 'org.hamcrest', name: 'hamcrest', version: '2.2' testImplementation group: 'org.jboss', name: 'jboss-vfs', version: '3.2.16.Final' - testImplementation group: 'org.springframework.boot', name: 'spring-boot-loader', version: '1.5.22.RELEASE' - - jmh group: 'org.springframework.boot', name: 'spring-boot-loader', version: '1.5.22.RELEASE' } jmh { diff --git a/telemetry/src/main/java/datadog/telemetry/dependency/Dependency.java b/telemetry/src/main/java/datadog/telemetry/dependency/Dependency.java index ce8a0e9d049..6a0d3b9b1ab 100644 --- a/telemetry/src/main/java/datadog/telemetry/dependency/Dependency.java +++ b/telemetry/src/main/java/datadog/telemetry/dependency/Dependency.java @@ -1,6 +1,5 @@ package datadog.telemetry.dependency; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; @@ -9,14 +8,11 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; -import java.util.Enumeration; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Properties; import java.util.jar.Attributes; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.jar.Manifest; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -75,59 +71,50 @@ public String toString() { + '}'; } - public static List fromMavenPom(JarFile jar) { - if (jar == null) { + public static List fromMavenPom( + final String jar, Map pomProperties) { + if (pomProperties == null) { return Collections.emptyList(); } - List dependencies = new ArrayList<>(1); - Enumeration entries = jar.entries(); - while (entries.hasMoreElements()) { - JarEntry jarEntry = entries.nextElement(); - String filename = jarEntry.getName(); - if (filename.endsWith("pom.properties")) { - try (InputStream is = jar.getInputStream(jarEntry)) { - if (is == null) { - return Collections.emptyList(); - } - Properties properties = new Properties(); - properties.load(is); - String groupId = properties.getProperty("groupId"); - String artifactId = properties.getProperty("artifactId"); - String version = properties.getProperty("version"); - String name = groupId + ":" + artifactId; - - if (groupId == null || artifactId == null || version == null) { - log.debug( - "'pom.properties' does not have all the required properties: " - + "jar={}, entry={}, groupId={}, artifactId={}, version={}", - jar.getName(), - jarEntry.getName(), - groupId, - artifactId, - version); - } else { - log.debug( - "dependency found in pom.properties: " - + "jar={}, entry={}, groupId={}, artifactId={}, version={}", - jar.getName(), - jarEntry.getName(), - groupId, - artifactId, - version); - dependencies.add( - new Dependency(name, version, new File(jar.getName()).getName(), null)); - } - } catch (IOException e) { - log.debug("unable to read 'pom.properties' file from {}", jar.getName(), e); - return Collections.emptyList(); - } + List dependencies = new ArrayList<>(pomProperties.size()); + for (final Map.Entry entry : pomProperties.entrySet()) { + final Properties properties = entry.getValue(); + final String groupId = properties.getProperty("groupId"); + final String artifactId = properties.getProperty("artifactId"); + final String version = properties.getProperty("version"); + final String name = groupId + ":" + artifactId; + + if (groupId == null || artifactId == null || version == null) { + log.debug( + "pom.properties does not have all the required properties: " + + "jar={}, entry={}, groupId={}, artifactId={}, version={}", + jar, + entry.getKey(), + groupId, + artifactId, + version); + } else { + log.debug( + "dependency found in pom.properties: " + + "jar={}, entry={}, groupId={}, artifactId={}, version={}", + jar, + entry.getKey(), + groupId, + artifactId, + version); + dependencies.add(new Dependency(name, version, jar, null)); } } return dependencies; } public static synchronized Dependency guessFallbackNoPom( - Manifest manifest, String source, InputStream is) throws IOException { + Attributes manifest, String source, InputStream is) throws IOException { + final int slashIndex = source.lastIndexOf('/'); + if (slashIndex >= 0) { + source = source.substring(slashIndex + 1); + } + String artifactId; String groupId = null; String version; @@ -140,12 +127,11 @@ public static synchronized Dependency guessFallbackNoPom( String bundleVersion = null; String implementationVersion = null; if (manifest != null) { - Attributes mainAttributes = manifest.getMainAttributes(); - bundleSymbolicName = mainAttributes.getValue("bundle-symbolicname"); - bundleName = mainAttributes.getValue("bundle-name"); - bundleVersion = mainAttributes.getValue("bundle-version"); - implementationTitle = mainAttributes.getValue("implementation-title"); - implementationVersion = mainAttributes.getValue("implementation-version"); + bundleSymbolicName = manifest.getValue("bundle-symbolicname"); + bundleName = manifest.getValue("bundle-name"); + bundleVersion = manifest.getValue("bundle-version"); + implementationTitle = manifest.getValue("implementation-title"); + implementationVersion = manifest.getValue("implementation-version"); } // Guess from file name @@ -204,7 +190,7 @@ public static synchronized Dependency guessFallbackNoPom( } if (md != null) { - // Compute hash for all dependencies that has no pom + // Compute hash for all dependencies that have no pom // No reliable version calculate hash and use any version md.reset(); is = new DigestInputStream(is, md); diff --git a/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java b/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java index e7ff1738c80..51cfc9726ec 100644 --- a/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java +++ b/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java @@ -1,16 +1,11 @@ package datadog.telemetry.dependency; import java.io.File; -import java.io.IOException; +import java.io.FileInputStream; import java.io.InputStream; -import java.net.JarURLConnection; import java.net.URI; -import java.net.URL; -import java.nio.file.Files; import java.util.Collections; import java.util.List; -import java.util.jar.JarFile; -import java.util.jar.Manifest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,20 +15,10 @@ public class DependencyResolver { private static final String JAR_SUFFIX = ".jar"; public static List resolve(URI uri) { - return extractDependenciesFromURI(uri); - } - - /** - * Identify library from a URI - * - * @param uri URI to a dependency - * @return dependency, or null if unable to qualify jar - */ - // package private for testing - static List extractDependenciesFromURI(URI uri) { - String scheme = uri.getScheme(); - List dependencies = Collections.emptyList(); + final String scheme = uri.getScheme(); try { + JarReader.Extracted metadata = null; + String path = null; if ("file".equals(scheme)) { File f; if (uri.isOpaque()) { @@ -41,89 +26,27 @@ static List extractDependenciesFromURI(URI uri) { } else { f = new File(uri); } - dependencies = extractDependenciesFromJar(f); - } else if ("jar".equals(scheme)) { - Dependency dependency = getNestedDependency(uri); - if (dependency != null) { - dependencies = Collections.singletonList(dependency); - } + path = f.getAbsolutePath(); + metadata = JarReader.readJarFile(path); + } else if ("jar".equals(scheme) && uri.getSchemeSpecificPart().startsWith("file:")) { + path = uri.getSchemeSpecificPart().substring("file:".length()); + metadata = JarReader.readNestedJarFile(path); + } else { + log.debug("unsupported dependency type: {}", uri); + return Collections.emptyList(); } - } catch (RuntimeException rte) { - log.debug("Failed to determine dependency for uri {}", uri, rte); - } - // TODO : moving jboss vfs here is probably a idea - // it might however require to do somme checks to make sure it's only applied to jboss - // and not any application server that also uses vfs:// locations - - return dependencies; - } - - /** - * Identify a library from a .jar file - * - * @param jar jar dependency - * @return detected dependency, {@code null} if unable to get dependency from jar - */ - static List extractDependenciesFromJar(File jar) { - if (!jar.exists()) { - log.debug("unable to find dependency {} (path does not exist)", jar); - return Collections.emptyList(); - } else if (!jar.getName().endsWith(JAR_SUFFIX)) { - log.debug("unsupported file dependency type : {}", jar); - return Collections.emptyList(); - } - - List dependencies = Collections.emptyList(); - try (JarFile file = new JarFile(jar, false /* no verify */)) { - - // Try to get from maven properties - dependencies = Dependency.fromMavenPom(file); - - // Try to guess from manifest or file name - if (dependencies.isEmpty()) { - try (InputStream is = Files.newInputStream(jar.toPath())) { - Manifest manifest = file.getManifest(); - dependencies = - Collections.singletonList(Dependency.guessFallbackNoPom(manifest, jar.getName(), is)); - } + final List dependencies = + Dependency.fromMavenPom(metadata.jarName, metadata.pomProperties); + if (!dependencies.isEmpty()) { + return dependencies; } - } catch (IOException e) { - log.debug("unable to read jar file {}", jar, e); - } - - return dependencies; - } - - /* for jar urls as handled by spring boot */ - static Dependency getNestedDependency(URI uri) { - String lastPart = null; - String fileName = null; - try { - URL url = uri.toURL(); - JarURLConnection jarConnection = (JarURLConnection) url.openConnection(); - Manifest manifest = jarConnection.getManifest(); - - JarFile jarFile = jarConnection.getJarFile(); - - // the !/ separator is hardcoded into JarURLConnection class - String jarFileName = jarFile.getName(); - int posSep = jarFileName.indexOf("!/"); - if (posSep == -1) { - log.debug("Unable to guess nested dependency for uri '{}': '!/' not found", uri); - return null; + try (final InputStream is = new FileInputStream(path)) { + return Collections.singletonList( + Dependency.guessFallbackNoPom(metadata.manifest, metadata.jarName, is)); } - lastPart = jarFileName.substring(posSep + 1); - fileName = lastPart.substring(lastPart.lastIndexOf("/") + 1); - - return Dependency.guessFallbackNoPom(manifest, fileName, jarConnection.getInputStream()); - } catch (Exception e) { - log.debug("unable to open nested jar manifest for {}", uri, e); + } catch (Throwable t) { + log.debug("Failed to determine dependency for uri {}", uri, t); } - log.debug( - "Unable to guess nested dependency for uri '{}', lastPart: '{}', fileName: '{}'", - uri, - lastPart, - fileName); - return null; + return Collections.emptyList(); } } diff --git a/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java b/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java new file mode 100644 index 00000000000..4d0616199b2 --- /dev/null +++ b/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java @@ -0,0 +1,87 @@ +package datadog.telemetry.dependency; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.NoSuchFileException; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.jar.Attributes; +import java.util.jar.JarFile; +import java.util.jar.JarInputStream; +import java.util.jar.Manifest; +import java.util.zip.ZipEntry; + +class JarReader { + static class Extracted { + final String jarName; + final Map pomProperties; + final Attributes manifest; + + public Extracted( + final String jarName, + final Map pomProperties, + final Attributes manifest) { + this.jarName = jarName; + this.pomProperties = pomProperties; + this.manifest = manifest; + } + } + + public static Extracted readJarFile(String jarPath) throws IOException { + try (final JarFile jar = new JarFile(jarPath, false /* no verify */)) { + final Map pomProperties = new HashMap<>(); + final Enumeration entries = jar.entries(); + while (entries.hasMoreElements()) { + final ZipEntry entry = entries.nextElement(); + if (entry.getName().endsWith("pom.properties")) { + try (final InputStream is = jar.getInputStream(entry)) { + final Properties properties = new Properties(); + properties.load(is); + pomProperties.put(entry.getName(), properties); + } + } + } + final Manifest manifest = jar.getManifest(); + final Attributes attributes = + (manifest == null) ? new Attributes() : manifest.getMainAttributes(); + return new Extracted(new File(jar.getName()).getName(), pomProperties, attributes); + } + } + + public static Extracted readNestedJarFile(final String jarPath) throws IOException { + final int sepIdx = jarPath.indexOf("!/"); + if (sepIdx == -1) { + throw new IllegalArgumentException("Invalid nested jar path: " + jarPath); + } + final String outerJarPath = jarPath.substring(0, sepIdx); + String innerJarPath = jarPath.substring(sepIdx + 2); + if (innerJarPath.endsWith("!/")) { + innerJarPath = innerJarPath.substring(0, innerJarPath.length() - 2); + } + try (final JarFile outerJar = new JarFile(outerJarPath, false /* no verify */)) { + final ZipEntry entry = outerJar.getEntry(innerJarPath); + if (entry == null) { + throw new NoSuchFileException("Nested jar not found: " + jarPath); + } + try (final InputStream is = outerJar.getInputStream(entry); + final JarInputStream innerJar = new JarInputStream(is, false /* no verify */)) { + final Map pomProperties = new HashMap<>(); + ZipEntry innerEntry; + while ((innerEntry = innerJar.getNextEntry()) != null) { + if (innerEntry.getName().endsWith("pom.properties")) { + final Properties properties = new Properties(); + properties.load(innerJar); + pomProperties.put(innerEntry.getName(), properties); + } + } + final Manifest manifest = innerJar.getManifest(); + final Attributes attributes = + (manifest == null) ? new Attributes() : manifest.getMainAttributes(); + return new Extracted(new File(innerJarPath).getName(), pomProperties, attributes); + } + } + } +} diff --git a/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy index 6ac756acfcf..9038f090105 100644 --- a/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy +++ b/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy @@ -105,7 +105,7 @@ class DependencyResolverSpecification extends DepSpecification { File temp = File.createTempFile('temp', '.zip') expect: - DependencyResolver.extractDependenciesFromJar(temp).isEmpty() + DependencyResolver.resolve(temp.toURI()).isEmpty() cleanup: temp.delete() @@ -117,7 +117,7 @@ class DependencyResolverSpecification extends DepSpecification { temp.delete() expect: - DependencyResolver.extractDependenciesFromJar(temp).isEmpty() + DependencyResolver.resolve(temp.toURI()).isEmpty() } void 'try to determine invalid jar lib'() throws IOException { @@ -126,22 +126,10 @@ class DependencyResolverSpecification extends DepSpecification { temp.write("just a text file") expect: - DependencyResolver.extractDependenciesFromJar(temp).isEmpty() - } - - void 'try to determine invalid jar lib'() throws IOException { - setup: - File temp = File.createTempFile('temp', '.jar') - temp.write("just a text file") - - expect: - DependencyResolver.getNestedDependency(temp.toURI()) == null + DependencyResolver.resolve(temp.toURI()).isEmpty() } void 'spring boot dependency'() throws IOException { - setup: - org.springframework.boot.loader.jar.JarFile.registerUrlProtocolHandler() - when: String zipPath = Classloader.classLoader.getResource('datadog/telemetry/dependencies/spring-boot-app.jar').path URI uri = new URI("jar:file:$zipPath!/BOOT-INF/lib/opentracing-util-0.33.0.jar!/") @@ -150,16 +138,13 @@ class DependencyResolverSpecification extends DepSpecification { then: dep != null - dep.name == 'opentracing-util' + dep.name == 'io.opentracing:opentracing-util' dep.version == '0.33.0' - dep.hash == '132630F17E198A1748F23CE33597EFDF4A807FB9' + dep.hash == null dep.source == 'opentracing-util-0.33.0.jar' } void 'fat jar with multiple pom.properties'() throws IOException { - setup: - org.springframework.boot.loader.jar.JarFile.registerUrlProtocolHandler() - when: URI uri = Classloader.classLoader.getResource('datadog/telemetry/dependencies/budgetapp.jar').toURI() @@ -171,9 +156,6 @@ class DependencyResolverSpecification extends DepSpecification { } void 'fat jar with two pom.properties'() throws IOException { - setup: - org.springframework.boot.loader.jar.JarFile.registerUrlProtocolHandler() - when: URI uri = Classloader.classLoader.getResource('datadog/telemetry/dependencies/budgetappreduced.jar').toURI() @@ -188,9 +170,6 @@ class DependencyResolverSpecification extends DepSpecification { } void 'fat jar with two pom.properties one of them bad'() throws IOException { - setup: - org.springframework.boot.loader.jar.JarFile.registerUrlProtocolHandler() - when: URI uri = Classloader.classLoader.getResource('datadog/telemetry/dependencies/budgetappreducedbadproperties.jar').toURI() @@ -215,7 +194,7 @@ class DependencyResolverSpecification extends DepSpecification { private static void knownJarCheck(Map opts) { File jarFile = getJar(opts['jarName']) - List deps = DependencyResolver.extractDependenciesFromJar(jarFile) + List deps = DependencyResolver.resolve(jarFile.toURI()) assert deps.size() == 1 Dependency dep = deps.get(0) diff --git a/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyServiceSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyServiceSpecification.groovy index 44144239c3f..1551d23f9e3 100644 --- a/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyServiceSpecification.groovy +++ b/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyServiceSpecification.groovy @@ -110,12 +110,11 @@ class DependencyServiceSpecification extends DepSpecification { depService.resolveOneDependency() then: - def set = depService.drainDeterminedDependencies() as Set - assertThat(set.size(), is(2)) - assertThat(set.first().name, is('cglib:cglib')) - assertThat(set.first().version, is('3.2.4')) - assertThat(set.last().name, is('org.yaml:snakeyaml')) - assertThat(set.last().version, is('1.17')) + def set = depService.drainDeterminedDependencies() as Set + set.size() == 2 + def map = set.collectEntries { [it.name, it] } + map['cglib:cglib'].version == '3.2.4' + map['org.yaml:snakeyaml'].version == '1.17' } void 'build dependency set from a small fat jar with one incorrect pom.properties'() { @@ -126,10 +125,10 @@ class DependencyServiceSpecification extends DepSpecification { depService.resolveOneDependency() then: - def set = depService.drainDeterminedDependencies() as Set - assertThat(set.size(), is(1)) - assertThat(set.first().name, is('org.yaml:snakeyaml')) - assertThat(set.first().version, is('1.17')) + def set = depService.drainDeterminedDependencies() as Set + set.size() == 1 + set.first().name == 'org.yaml:snakeyaml' + set.first().version == '1.17' } void 'transformer invalid code source'() throws IllegalClassFormatException, MalformedURLException { diff --git a/telemetry/src/test/groovy/datadog/telemetry/dependency/JarReaderSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/dependency/JarReaderSpecification.groovy new file mode 100644 index 00000000000..cf666827445 --- /dev/null +++ b/telemetry/src/test/groovy/datadog/telemetry/dependency/JarReaderSpecification.groovy @@ -0,0 +1,126 @@ +package datadog.telemetry.dependency + +class JarReaderSpecification extends DepSpecification { + + void 'read plain jar with manifest and no pom.properties'() { + given: + String jarPath = getJar("bson-4.2.0.jar").getAbsolutePath() + + when: + def result = JarReader.readJarFile(jarPath) + + then: + result.jarName == "bson-4.2.0.jar" + result.pomProperties.isEmpty() + result.manifest != null + result.manifest.getValue("Bundle-Name") == "bson" + } + + void 'read jar without manifest'() { + given: + String jarPath = getJar("groovy-no-manifest-info.jar").getAbsolutePath() + + when: + def result = JarReader.readJarFile(jarPath) + + then: + result.jarName == "groovy-no-manifest-info.jar" + result.pomProperties.isEmpty() + result.manifest != null + result.manifest.isEmpty() + } + + void 'read plain jar with manifest and pom.properties'() { + given: + String jarPath = getJar("commons-logging-1.2.jar").getAbsolutePath() + + when: + def result = JarReader.readJarFile(jarPath) + + then: + result.jarName == "commons-logging-1.2.jar" + result.pomProperties.size() == 1 + def properties = result.pomProperties['META-INF/maven/commons-logging/commons-logging/pom.properties'] + properties != null + properties.groupId == "commons-logging" + properties.artifactId == "commons-logging" + properties.version == "1.2" + result.manifest != null + result.manifest.getValue("Bundle-Name") == "Apache Commons Logging" + } + + void 'read nested jar'() { + given: + String outerPath = getJar("spring-boot-app.jar").getAbsolutePath() + String jarPath = "$outerPath!/BOOT-INF/lib/opentracing-util-0.33.0.jar" + + when: + def result = JarReader.readNestedJarFile(jarPath) + + then: + result.jarName == "opentracing-util-0.33.0.jar" + result.pomProperties.size() == 1 + def properties = result.pomProperties['META-INF/maven/io.opentracing/opentracing-util/pom.properties'] + properties.groupId == "io.opentracing" + properties.artifactId == "opentracing-util" + properties.version == "0.33.0" + result.manifest != null + result.manifest.getValue("Automatic-Module-Name") == "io.opentracing.util" + } + + void 'non-existent simple jar'() { + given: + String jarPath = "non-existent.jar" + + when: + JarReader.readJarFile(jarPath) + + then: + thrown(IOException) + } + + void 'non-existent outer jar for nested jar'() { + given: + String jarPath = "non-existent.jar!/BOOT-INF/lib/opentracing-util-0.33.0.jar!/" + + when: + JarReader.readNestedJarFile(jarPath) + + then: + thrown(IOException) + } + + void 'non-existent inner jar for nested jar'() { + given: + String outerPath = getJar("spring-boot-app.jar").getAbsolutePath() + String jarPath = "$outerPath!/BOOT-INF/lib/non-existent.jar" + + when: + JarReader.readNestedJarFile(jarPath) + + then: + thrown(IOException) + } + + void 'doubly nested jar path'() { + given: + String jarPath = "non-existent.jar!/BOOT-INF/lib/opentracing-util-0.33.0.jar!/third" + + when: + JarReader.readNestedJarFile(jarPath) + + then: + thrown(IOException) + } + + void 'lack of nested jar path'() { + given: + String jarPath = "non-existent.jar" + + when: + JarReader.readNestedJarFile(jarPath) + + then: + thrown(IllegalArgumentException) + } +}