-
Notifications
You must be signed in to change notification settings - Fork 69
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Clean up raster disposing. Clean up duplicated expressions.
- Loading branch information
milos.colic
committed
Oct 4, 2023
1 parent
ddab8cc
commit 43e483f
Showing
52 changed files
with
292 additions
and
542 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
48 changes: 48 additions & 0 deletions
48
src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/OverlappingTiles.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package com.databricks.labs.mosaic.core.raster.operator.retile | ||
|
|
||
| import com.databricks.labs.mosaic.core.raster.MosaicRaster | ||
| import com.databricks.labs.mosaic.core.raster.operator.gdal.GDALTranslate | ||
| import com.databricks.labs.mosaic.utils.PathUtils | ||
|
|
||
| import scala.collection.immutable | ||
|
|
||
| object OverlappingTiles { | ||
|
|
||
| def reTile( | ||
| raster: MosaicRaster, | ||
| tileWidth: Int, | ||
| tileHeight: Int, | ||
| overlapPercentage: Int | ||
| ): immutable.Seq[MosaicRaster] = { | ||
| val (xSize, ySize) = raster.getDimensions | ||
|
|
||
| val overlapWidth = Math.ceil(tileWidth * overlapPercentage / 100.0).toInt | ||
| val overlapHeight = Math.ceil(tileHeight * overlapPercentage / 100.0).toInt | ||
|
|
||
| val tiles = for (i <- 0 until xSize by (tileWidth - overlapWidth)) yield { | ||
| for (j <- 0 until ySize by (tileHeight - overlapHeight)) yield { | ||
| val xOff = if (i == 0) i else i - 1 | ||
| val yOff = if (j == 0) j else j - 1 | ||
| val width = Math.min(tileWidth, xSize - i) + 1 | ||
| val height = Math.min(tileHeight, ySize - j) + 1 | ||
|
|
||
| val uuid = java.util.UUID.randomUUID.toString | ||
| val rasterPath = PathUtils.createTmpFilePath(uuid, "tif") | ||
|
|
||
| val result = GDALTranslate.executeTranslate( | ||
| rasterPath, | ||
| isTemp = true, | ||
| raster, | ||
| command = s"gdal_translate -srcwin $xOff $yOff $width $height" | ||
| ) | ||
|
|
||
| result.flushCache() | ||
| } | ||
| } | ||
|
|
||
| tiles.flatten | ||
|
|
||
|
|
||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
73 changes: 73 additions & 0 deletions
73
src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBox.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| package com.databricks.labs.mosaic.expressions.raster | ||
|
|
||
| import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI | ||
| import com.databricks.labs.mosaic.core.raster.MosaicRaster | ||
| import com.databricks.labs.mosaic.core.types.model.GeometryTypeEnum | ||
| import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} | ||
| import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression | ||
| import com.databricks.labs.mosaic.functions.MosaicExpressionConfig | ||
| import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder | ||
| import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback | ||
| import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} | ||
| import org.apache.spark.sql.types._ | ||
|
|
||
| /** Returns the world coordinates of the raster (x,y) pixel. */ | ||
| case class RST_BoundingBox( | ||
| raster: Expression, | ||
| expressionConfig: MosaicExpressionConfig | ||
| ) extends RasterExpression[RST_RasterToWorldCoord](raster, BinaryType, returnsRaster = false, expressionConfig = expressionConfig) | ||
| with NullIntolerant | ||
| with CodegenFallback { | ||
|
|
||
| /** | ||
| * The function to be overridden by the extending class. It is called when | ||
| * the expression is evaluated. It provides the raster to the expression. | ||
| * It abstracts spark serialization from the caller. | ||
| * | ||
| * @param raster | ||
| * The raster to be used. | ||
| * @return | ||
| * The result of the expression. | ||
| */ | ||
| override def rasterTransform(raster: MosaicRaster): Any = { | ||
| val gt = raster.getRaster.GetGeoTransform() | ||
| val (originX, originY) = rasterAPI.toWorldCoord(gt, 0, 0) | ||
| val (endX, endY) = rasterAPI.toWorldCoord(gt, raster.xSize, raster.ySize) | ||
| val geometryAPI = GeometryAPI(expressionConfig.getGeometryAPI) | ||
| val bboxPolygon = geometryAPI.geometry( | ||
| Seq( | ||
| Seq(originX, originY), | ||
| Seq(originX, endY), | ||
| Seq(endX, endY), | ||
| Seq(endX, originY), | ||
| Seq(originX, originY) | ||
| ).map(geometryAPI.fromCoords), | ||
| GeometryTypeEnum.POLYGON | ||
| ) | ||
| bboxPolygon.toWKB | ||
| } | ||
|
|
||
| } | ||
|
|
||
| /** Expression info required for the expression registration for spark SQL. */ | ||
| object RST_BoundingBox extends WithExpressionInfo { | ||
|
|
||
| override def name: String = "rst_boundingbox" | ||
|
|
||
| override def usage: String = | ||
| """ | ||
| |_FUNC_(expr1) - Returns the bounding box of the raster. | ||
| |""".stripMargin | ||
|
|
||
| override def example: String = | ||
| """ | ||
| | Examples: | ||
| | > SELECT _FUNC_(a, b, c); | ||
| | (11.2, 12.3) | ||
| | """.stripMargin | ||
|
|
||
| override def builder(expressionConfig: MosaicExpressionConfig): FunctionBuilder = { | ||
| GenericExpressionFactory.getBaseBuilder[RST_RasterToWorldCoord](3, expressionConfig) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.