Initial commit

lib/ex_data_gov_u_a/application.ex

@@ -0,0 +1,18 @@
+defmodule ExDataGovUA.Application do
+  # See https://hexdocs.pm/elixir/Application.html
+  # for more information on OTP Applications
+  @moduledoc false
+
+  use Application
+
+  def start(_type, _args) do
+    children = [
+      # {Finch, name: ExDataGovUA.Finch}
+    ]
+
+    # See https://hexdocs.pm/elixir/Supervisor.html
+    # for other strategies and supported options
+    opts = [strategy: :one_for_one, name: ExDataGovUA.Supervisor]
+    Supervisor.start_link(children, opts)
+  end
+end

lib/ex_data_gov_u_a/archivers/base.ex

@@ -0,0 +1,7 @@
+defmodule ExDataGovUA.Archivers.Base do
+  @type stream :: Stream.t()
+
+  @callback deflate(stream) :: stream
+
+  @callback inflate(stream) :: stream
+end

lib/ex_data_gov_u_a/archivers/gz.ex

@@ -0,0 +1,19 @@
+defmodule ExDataGovUA.Archivers.Gz do
+  @behaviour ExDataGovUA.Archivers.Base
+
+  @type input :: Stream.t()
+  @type output :: Stream.t()
+
+  @spec deflate(input) :: output
+  @doc """
+    Pack data to archive
+  """
+  def deflate(stream), do: stream |> StreamGzip.gzip
+
+
+  @spec inflate(input) :: output
+  @doc """
+    Extract data from archive
+  """
+  def inflate(stream), do: stream |> StreamGzip.gunzip
+end

lib/ex_data_gov_u_a/commons/http_stream.ex

@@ -0,0 +1,93 @@
+defmodule ExDataGovUA.Commons.HttpStream do
+  def get(url,emit_end\\false) do
+
+    Stream.resource(
+      fn -> start_fun(url) end,
+
+      # next_fun (multi caluses)
+      fn
+        %HTTPoison.AsyncResponse{}=resp ->
+          handle_async_resp(resp,emit_end)
+
+        #last accumulator when emitting :end
+        {:end, resp}->
+          {:halt, resp}
+      end,
+
+      fn %HTTPoison.AsyncResponse{id: id} ->
+        IO.puts("END_FUN")
+        :hackney.stop_async(id)
+      end
+    )
+  end
+
+
+  defp start_fun(url) do
+    HTTPoison.get!(url,%{}, [stream_to: self(), async: :once])
+  end
+
+
+  defp handle_async_resp(%HTTPoison.AsyncResponse{id: id}=resp,emit_end) do
+    receive do
+      %HTTPoison.AsyncStatus{id: ^id, code: code}->
+        IO.inspect(code, label: "STATUS: ")
+        HTTPoison.stream_next(resp)
+        {[], resp}
+      %HTTPoison.AsyncHeaders{id: ^id, headers: headers}->
+        IO.inspect(headers, label: "HEADERS: ")
+        HTTPoison.stream_next(resp)
+        {[], resp}
+      %HTTPoison.AsyncChunk{id: ^id, chunk: chunk}->
+        HTTPoison.stream_next(resp)
+        # :erlang.garbage_collect()
+        {[chunk], resp}
+      %HTTPoison.AsyncEnd{id: ^id}->
+        if emit_end do
+          {[:end], {:end, resp}}
+        else
+          {:halt, resp}
+        end
+    after
+      5_000 -> raise "receive timeout"
+    end
+  end
+
+  def lines(enum), do: lines(enum, :string_split)
+
+  def lines(enum, :next_lines) do
+    enum
+    |> Stream.transform("",&next_lines/2)
+  end
+
+  def lines(enum, :string_split) do
+    enum
+    |> Stream.transform("",fn
+      :end, acc ->
+        {[acc],""}
+      chunk, acc ->
+        [last_line | lines] =
+          String.split(acc <> chunk,"\n")
+          |> Enum.reverse()
+        {Enum.reverse(lines),last_line}
+    end)
+  end
+
+
+  defp next_lines(:end,prev), do: {[prev], ""}
+  defp next_lines(chunk,current_line) do
+    # :erlang.garbage_collect()
+    next_lines(chunk,current_line,[])
+  end
+
+  defp next_lines(<<"\n"::utf8, rest::binary>>,current_line,lines) do
+    next_lines(rest,"",[<<current_line::binary,"\n"::utf8>> | lines])
+  end
+
+  defp next_lines(<<c::utf8, rest::binary>>,current_line,lines) do
+    next_lines(rest,<<current_line::binary, c::utf8>>,lines)
+  end
+
+  defp next_lines(<<>>,current_line,lines) do
+    {Enum.reverse(lines), current_line}
+  end
+end

lib/ex_data_gov_u_a/decoders/base.ex

@@ -0,0 +1,9 @@
+defmodule ExDataGovUA.Decoders.Base do
+  @type data :: map()
+  @type status :: Atom.t()
+  @type async_response :: Stream.t()
+  @type sync_response :: {status, data}
+  @type body :: String.t() | Stream.t()
+
+  @callback decode(body) :: async_response | sync_response
+end

lib/ex_data_gov_u_a/decoders/jason_decoder.ex

@@ -0,0 +1,8 @@
+defmodule ExDataGovUA.Decoders.JasonDecoder do
+  @behaviour ExDataGovUA.Decoders.Base
+
+  @impl true
+  def decode(data) do
+    Jason.decode(data, [keys: :atoms])
+  end
+end

lib/ex_data_gov_u_a/decoders/jaxon_decoder.ex

@@ -0,0 +1,9 @@
+defmodule ExDataGovUA.Decoders.JaxonDecoder do
+  @behaviour ExDataGovUA.Decoders.Base
+
+  @impl true
+  def decode(stream) do
+    stream
+    |> Jaxon.Stream.from_enumerable()
+  end
+end

lib/ex_data_gov_u_a/fetchers/base.ex

@@ -0,0 +1,8 @@
+defmodule ExDataGovUA.Fetchers.Base do
+  @type data :: map()
+  @type status :: Atom.t()
+  @type url :: String.t()
+  @type method :: Atom.t()
+
+  @callback fetch(method, url) :: {status, data}
+end

lib/ex_data_gov_u_a/fetchers/finch_fetcher.ex

@@ -0,0 +1,19 @@
+defmodule ExDataGovUA.Fetchers.FinchFetcher do
+  @behaviour ExDataGovUA.Fetchers.Base
+
+  @impl true
+  def fetch(method, url) do
+    start()
+    with request <- Finch.build(method, url),
+         {:ok, %Finch.Response{body: body}} <- Finch.request(request, MyFinchFetcher) do
+      {:ok, body}
+    end
+  end
+
+  defp start() do
+    case Finch.start_link(name: MyFinchFetcher) do
+      {:ok, pid} -> pid
+      {:error, {:already_started, pid}} -> pid
+    end
+  end
+end

lib/ex_data_gov_u_a/package/base.ex

@@ -0,0 +1,3 @@
+defmodule ExDataGovUA.Package.Base do
+  defstruct [:id, :meta]
+end

lib/ex_data_gov_u_a/package/meta/base.ex

@@ -0,0 +1,33 @@
+defmodule ExDataGovUA.Package.Meta.Base do
+  defstruct [
+    :profile,
+    :name,
+    :contributors,
+    :created,
+    :title,
+    :keywords,
+    :version,
+    :licenses,
+    :homepage,
+    :id,
+    :resources,
+    :description
+  ]
+
+  @type t :: %__MODULE__{
+    profile: String.t(),
+    name: String.t(),
+    contributors: List.t(),
+    created: String.t(),
+    title: String.t(),
+    keywords: List.t(),
+    version: String.t(),
+    licenses: List.t(),
+    homepage: String.t(),
+    id: String.t(),
+    resources: List.t(),
+    description: String.t()
+  }
+
+  use ExConstructor
+end

lib/ex_data_gov_u_a/package/meta/parser.ex

@@ -0,0 +1,30 @@
+defmodule ExDataGovUA.Package.Meta.Parser do
+  @default_fetcher Application.get_env(:ex_datagovua, :default_fetcher)
+  @default_decoder Application.get_env(:ex_datagovua, :default_decoder)
+
+  @type result :: %ExDataGovUA.Package.Meta.Base{}
+  @type id :: String.t()
+  @type url :: String.t()
+  @type fetcher :: Atom.t()
+  @type decoder :: Atom.t()
+
+  @doc """
+  Return meta information URL by data package ID
+  """
+  @spec url(id) :: url
+  def url(id) do
+    "https://data.gov.ua/api/3/action/package_show?id=#{id}"
+  end
+
+  @doc """
+  Return meta information (ExDataGovUA.Package.Meta.Base) by data package ID
+  """
+  @spec parse(id, fetcher, decoder) :: %ExDataGovUA.Package.Meta.Base{}
+  def parse(id, fetcher \\ @default_fetcher, decoder \\ @default_decoder) do
+    with {:ok, body} <- fetcher.fetch(:get, url(id)),
+         {:ok, map} <- decoder.decode(body),
+         result <- ExDataGovUA.Package.Meta.Base.new(map.result) do
+      result
+    end
+  end
+end

lib/ex_data_gov_u_a/resource/base.ex

@@ -0,0 +1,11 @@
+defmodule ExDataGovUA.Resource.Base do
+  def download(url, _filename) do
+    url
+    |> ExDataGovUA.Streamers.HttpStreamer.get()
+    |> ExDataGovUA.Archivers.Gz.inflate()
+    |> ExDataGovUA.Decoders.JaxonDecoder.decode()
+    |> Enum.to_list()
+    # |> Stream.run()
+    # |> ExDataGovUA.Streamers.FileStreamer.export(filename)
+  end
+end

lib/ex_data_gov_u_a/streamers/file_streamer.ex

@@ -0,0 +1,14 @@
+defmodule ExDataGovUA.Streamers.FileStreamer do
+  @chunk_size 2048
+  @spec export(Stream.t(), String.t()) :: Atom.t()
+  def export(stream, filename) do
+    stream
+    |> Stream.into(File.stream!(filename, [], @chunk_size))
+  end
+
+  @spec import(String.t()) :: File.Stream.t()
+  def import(filename) do
+    filename
+    |> File.stream!([], @chunk_size)
+  end
+end

lib/ex_data_gov_u_a/streamers/http_streamer.ex

@@ -0,0 +1,8 @@
+defmodule ExDataGovUA.Streamers.HttpStreamer do
+  alias ExDataGovUA.Commons.HttpStream
+
+  def get(url) do
+    url
+    |> HttpStream.get()
+  end
+end

lib/ex_datagovua.ex

@@ -0,0 +1,18 @@
+defmodule ExDataGovUA do
+  @moduledoc """
+  Documentation for `ExDatagovua`.
+  """
+
+  @doc """
+  Hello world.
+
+  ## Examples
+
+      iex> ExDatagovua.hello()
+      :world
+
+  """
+  def hello do
+    :world
+  end
+end