Initial commit

.formatter.exs

@@ -0,0 +1,4 @@
+[
+  import_deps: [:phoenix],
+  inputs: ["*.{ex,exs}", "{config,lib,test}/**/*.{ex,exs}"]
+]

.gitignore

@@ -0,0 +1,24 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where 3rd-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+exapisd-*.tar
+

README.md

@@ -0,0 +1,42 @@
+# Exapisd
+A simple web API for serving data from json, SQLite or csv static files to json.
+
+## Installation and setup
+
+To start Exapisd:
+
+  * Run `git clone...` from the repository
+  * Install dependencies with `mix deps.get`
+  * Run `mix compile`
+  * Set the repository path `repo` in `config.exs` 
+  * Start Phoenix endpoint with `mix phx.server` or inside IEx with `iex -S mix phx.server`
+
+Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.
+
+Ready to run in production? Please [check Phoenix deployment guides](https://hexdocs.pm/phoenix/deployment.html).
+
+## Usage
+The API's web links serves the data as it is structured on the file repository set on the server.
+### Example
+Considering the repository path is `./temp/repo` and in the sub-directory `/test/` there is `test.json` file:
+`./temp/repo/json/test.json`. Then the web API link will be http://localhost:4000/json
+The repository structure can be nested as well: for `./temp/repo/json/another/test2.json` you get http://localhost:4000/json/another
+
+Other examples:
+ - `./temp/repo/sqlite/testsqlite.db` -> http://localhost:4000/sqlite
+ - `./temp/repo/csv/large/foe.csv` -> http://localhost:4000/csv/large
+
+The application serves the first alphabetically file found in it. If you have both `bar.json` and `foe.csv` in the same directory, the `bar.json` will be fetched. 
+
+## TODO
+ - tests/benchmarks
+ - add support for xls, tsv, xml, unstructured
+ - handle errors related to corrupt files (help appreciated!) in a better way.
+
+## Who is using it?
+This application is in production (used internally) at the Joint Research Center of the European Commission.
+
+## License
+Copyright © 2022 Quda Theo.
+
+Exapisd source code is released under **[AGPL-3.0-or-later](https://www.gnu.org/licenses/agpl-3.0.html)**.

config/config.exs

@@ -0,0 +1,31 @@
+# This file is responsible for configuring your application
+# and its dependencies with the aid of the Config module.
+#
+# This configuration file is loaded before any dependency and
+# is restricted to this project.
+
+# General application configuration
+import Config
+
+# Configures the endpoint
+config :exapisd, ExapisdWeb.Endpoint,
+  url: [host: "localhost"],
+  render_errors: [view: ExapisdWeb.ErrorView, accepts: ~w(json), layout: false],
+  pubsub_server: Exapisd.PubSub,
+  live_view: [signing_salt: "Rn2bgSyl"]
+
+# Configures Elixir's Logger
+config :logger, :console,
+  format: "$time $metadata[$level] $message\n",
+  metadata: [:request_id]
+
+# Use Jason for JSON parsing in Phoenix
+config :phoenix, :json_library, Jason
+
+config :exapisd,
+  repo: "../repo",
+  switch_threshold: 1
+
+# Import environment specific config. This must remain at the bottom
+# of this file so it overrides the configuration defined above.
+import_config "#{config_env()}.exs"

config/dev.exs

@@ -0,0 +1,51 @@
+import Config
+
+# For development, we disable any cache and enable
+# debugging and code reloading.
+#
+# The watchers configuration can be used to run external
+# watchers to your application. For example, we use it
+# with esbuild to bundle .js and .css sources.
+config :exapisd, ExapisdWeb.Endpoint,
+  # Binding to loopback ipv4 address prevents access from other machines.
+  # Change to `ip: {0, 0, 0, 0}` to allow access from other machines.
+  http: [ip: {127, 0, 0, 1}, port: 4000],
+  check_origin: false,
+  code_reloader: true,
+  debug_errors: true,
+  secret_key_base: "3tM+mHaNaTUDA2HXLGZI7ttqQ0xSpqzkbXr200gmnMIKvv36vbM/d79NGT90kBjm",
+  watchers: []
+
+# ## SSL Support
+#
+# In order to use HTTPS in development, a self-signed
+# certificate can be generated by running the following
+# Mix task:
+#
+#     mix phx.gen.cert
+#
+# Note that this task requires Erlang/OTP 20 or later.
+# Run `mix help phx.gen.cert` for more information.
+#
+# The `http:` config above can be replaced with:
+#
+#     https: [
+#       port: 4001,
+#       cipher_suite: :strong,
+#       keyfile: "priv/cert/selfsigned_key.pem",
+#       certfile: "priv/cert/selfsigned.pem"
+#     ],
+#
+# If desired, both `http:` and `https:` keys can be
+# configured to run both http and https servers on
+# different ports.
+
+# Do not include metadata nor timestamps in development logs
+config :logger, :console, format: "[$level] $message\n"
+
+# Set a higher stacktrace during development. Avoid configuring such
+# in production as building large stacktraces may be expensive.
+config :phoenix, :stacktrace_depth, 20
+
+# Initialize plugs at runtime for faster development compilation
+config :phoenix, :plug_init_mode, :runtime

config/prod.exs

@@ -0,0 +1,51 @@
+import Config
+
+# For production, don't forget to configure the url host
+# to something meaningful, Phoenix uses this information
+# when generating URLs.
+#
+# Note we also include the path to a cache manifest
+# containing the digested version of static files. This
+# manifest is generated by the `mix phx.digest` task,
+# which you should run after static files are built and
+# before starting your production server.
+config :exapisd, ExapisdWeb.Endpoint,
+  url: [host: "example.com", port: 80],
+  cache_static_manifest: "priv/static/cache_manifest.json"
+
+# Do not print debug messages in production
+config :logger, level: :info
+
+# ## SSL Support
+#
+# To get SSL working, you will need to add the `https` key
+# to the previous section and set your `:url` port to 443:
+#
+#     config :exapisd, ExapisdWeb.Endpoint,
+#       ...,
+#       url: [host: "example.com", port: 443],
+#       https: [
+#         ...,
+#         port: 443,
+#         cipher_suite: :strong,
+#         keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"),
+#         certfile: System.get_env("SOME_APP_SSL_CERT_PATH")
+#       ]
+#
+# The `cipher_suite` is set to `:strong` to support only the
+# latest and more secure SSL ciphers. This means old browsers
+# and clients may not be supported. You can set it to
+# `:compatible` for wider support.
+#
+# `:keyfile` and `:certfile` expect an absolute path to the key
+# and cert in disk or a relative path inside priv, for example
+# "priv/ssl/server.key". For all supported SSL configuration
+# options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1
+#
+# We also recommend setting `force_ssl` in your endpoint, ensuring
+# no data is ever sent via http, always redirecting to https:
+#
+#     config :exapisd, ExapisdWeb.Endpoint,
+#       force_ssl: [hsts: true]
+#
+# Check `Plug.SSL` for all available options in `force_ssl`.

config/runtime.exs

@@ -0,0 +1,42 @@
+import Config
+
+# config/runtime.exs is executed for all environments, including
+# during releases. It is executed after compilation and before the
+# system starts, so it is typically used to load production configuration
+# and secrets from environment variables or elsewhere. Do not define
+# any compile-time configuration in here, as it won't be applied.
+# The block below contains prod specific runtime configuration.
+if config_env() == :prod do
+  # The secret key base is used to sign/encrypt cookies and other secrets.
+  # A default value is used in config/dev.exs and config/test.exs but you
+  # want to use a different value for prod and you most likely don't want
+  # to check this value into version control, so we use an environment
+  # variable instead.
+  secret_key_base =
+    System.get_env("SECRET_KEY_BASE") ||
+      raise """
+      environment variable SECRET_KEY_BASE is missing.
+      You can generate one by calling: mix phx.gen.secret
+      """
+
+  config :exapisd, ExapisdWeb.Endpoint,
+    http: [
+      # Enable IPv6 and bind on all interfaces.
+      # Set it to  {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
+      # See the documentation on https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html
+      # for details about using IPv6 vs IPv4 and loopback vs public addresses.
+      ip: {0, 0, 0, 0, 0, 0, 0, 0},
+      port: String.to_integer(System.get_env("PORT") || "4000")
+    ],
+    secret_key_base: secret_key_base
+
+  # ## Using releases
+  #
+  # If you are doing OTP releases, you need to instruct Phoenix
+  # to start each relevant endpoint:
+  #
+  #     config :exapisd, ExapisdWeb.Endpoint, server: true
+  #
+  # Then you can assemble a release by calling `mix release`.
+  # See `mix help release` for more information.
+end

config/test.exs

@@ -0,0 +1,14 @@
+import Config
+
+# We don't run a server during test. If one is required,
+# you can enable the server option below.
+config :exapisd, ExapisdWeb.Endpoint,
+  http: [ip: {127, 0, 0, 1}, port: 4002],
+  secret_key_base: "JglI2NvTI+aDNkTvnU5MGn3ihI+Phb6uelv2dYMl+VfecN/AtDQ2y2bCDj1ZUfLg",
+  server: false
+
+# Print only warnings and errors during test
+config :logger, level: :warn
+
+# Initialize plugs at runtime for faster test compilation
+config :phoenix, :plug_init_mode, :runtime

curl.txt

@@ -0,0 +1 @@
+curl http://localhost:4000/csv/l --noproxy localhost -o NUL

lib/exapisd.ex

@@ -0,0 +1,9 @@
+defmodule Exapisd do
+  @moduledoc """
+  Exapisd keeps the contexts that define your domain
+  and business logic.
+
+  Contexts are also responsible for managing your data, regardless
+  if it comes from the database, an external API or others.
+  """
+end

lib/exapisd/application.ex

@@ -0,0 +1,34 @@
+defmodule Exapisd.Application do
+  # See https://hexdocs.pm/elixir/Application.html
+  # for more information on OTP Applications
+  @moduledoc false
+
+  use Application
+
+  @impl true
+  def start(_type, _args) do
+    children = [
+      # Start the Telemetry supervisor
+      ExapisdWeb.Telemetry,
+      # Start the PubSub system
+      {Phoenix.PubSub, name: Exapisd.PubSub},
+      # Start the Endpoint (http/https)
+      ExapisdWeb.Endpoint
+      # Start a worker by calling: Exapisd.Worker.start_link(arg)
+      # {Exapisd.Worker, arg}
+    ]
+
+    # See https://hexdocs.pm/elixir/Supervisor.html
+    # for other strategies and supported options
+    opts = [strategy: :one_for_one, name: Exapisd.Supervisor]
+    Supervisor.start_link(children, opts)
+  end
+
+  # Tell Phoenix to update the endpoint configuration
+  # whenever the application is updated.
+  @impl true
+  def config_change(changed, _new, removed) do
+    ExapisdWeb.Endpoint.config_change(changed, removed)
+    :ok
+  end
+end

lib/exapisd/backend.ex

@@ -0,0 +1,62 @@
+defmodule Exapisd.Backend do
+  # import Exapisd.Repo
+  import Exapisd.WithFile
+  import Exapisd.WithStream
+
+  @root Application.get_env(:exapisd, :repo)
+  @t Application.get_env(:exapisd, :switch_threshold) #in MB
+
+  def path(url) do
+    fullpath = Path.expand(@root <> url)
+    if File.dir?(fullpath) do
+      {:ok, files} = File.ls(fullpath)
+      files(files, fullpath)
+    else
+      {:error, 404, "Path not found"}
+    end
+  end
+
+  defp files(files, fullpath) do
+    Enum.filter(files, fn f ->
+      Path.extname(f) != "" &&
+        Enum.member?([".csv", ".json", ".sqlite", ".db", ".sqlite3"], Path.extname(f))
+    end)
+    |> Enum.take(1)
+    |> switchBySize(fullpath <> "/")
+  end
+
+  defp switchBySize([f], fullpath) do
+    file = (fullpath <> f) |> Path.expand(__DIR__)
+
+    case File.stat(file) do
+      {:ok, %{size: size}} -> identify(file, size)
+      {:error, reason} -> {:error, 500, reason}
+    end
+  end
+
+  defp switchBySize([], _fullpath) do
+    {:error, 404, "no file in the path"}
+  end
+
+  defp identify(file, size) when size < @t do
+    case Path.extname(file) do
+      ".json" -> js(file)
+      ".csv" -> csv(file)
+      ".db" -> sqlite(file)
+      ".sqlite" -> sqlite(file)
+      ".sqlite3" -> sqlite(file)
+      _ -> {:error, 404, "No suitable file"}
+    end
+  end
+
+  defp identify(file, size) when size >= @t do
+    case Path.extname(file) do
+      ".json" -> json_stream(file)
+      ".csv" -> csv_stream(file)
+      ".db" -> sqlite_stream(file)
+      ".sqlite" -> sqlite_stream(file)
+      ".sqlite3" -> sqlite_stream(file)
+      _ -> {:error, 404, "No suitable file"}
+    end
+  end
+end