Minimal worked solution

This commit is contained in:
Danil Negrienko 2020-04-21 04:34:26 +03:00
parent b45a818fa4
commit 2899e4281e
8 changed files with 213 additions and 24 deletions

View File

@ -6,6 +6,10 @@ config :logger, :console,
metadata: [:request_id] metadata: [:request_id]
config :translator, config :translator,
input_folder: "input",
output_folder: "output",
base_folder: "priv",
translators: [Translator.Yandex],
parsers: [Translator.Parser.YAML, Translator.Parser.JSON], parsers: [Translator.Parser.YAML, Translator.Parser.JSON],
availiable_languages: ~w(en zh ar es hi de ru be uk kk bn pl pt it fr) availiable_languages: ~w(en zh ar es hi de ru be uk kk bn pl pt it fr)

View File

@ -1,28 +1,79 @@
defmodule Translator.Batcher do defmodule Translator.Batcher do
alias Translator.Languages alias Translator.{Languages, Parser}
def process(batch) do
# ,:ok <- translate_and_save(loaded)
with {:ok, formed} <- parse(batch),
{:ok, loaded} <- load(formed),
:ok <- translate_and_save(loaded) do
loaded
end
end
defp base_folder(), do: Application.get_env(:translator, :base_folder, "priv")
defp output_folder(), do: Application.get_env(:translator, :output_folder, "output")
defp input_folder(), do: Application.get_env(:translator, :input_folder, "input")
defp input_path(), do: Path.join(base_folder(), input_folder())
defp output_path(), do: Path.join(base_folder(), output_folder())
defp load(batch) do
loaded =
batch.files
|> Enum.reduce([], fn filename, acc ->
path = Path.join(batch.from, filename)
{:ok, content} = Parser.load(path)
[{filename, content} | acc]
end)
|> Map.new()
{:ok, Map.put(batch, :loaded, loaded)}
end
@spec parse(String.t()) :: {:ok, map} | {:error, String.t()} @spec parse(String.t()) :: {:ok, map} | {:error, String.t()}
def parse(path) do defp parse(batch) do
batch = Path.basename(path) from =
Path.join([input_path(), batch, Languages.pattern()])
folder =
Path.join(path, Languages.pattern())
|> Path.wildcard() |> Path.wildcard()
|> List.first() |> List.first()
case folder do case from do
nil -> nil ->
{:error, "Couldn't find input folder in batch '#{path}'"} {:error, "Couldn't find input folder in batch at '#{from}'"}
_ -> _ ->
files =
Path.wildcard(Path.join(from, "**"))
|> Enum.map(fn path -> Path.relative_to(path, from) end)
{:ok, {:ok,
%{ %{
name: batch, batch: batch,
folder: path, from: from,
input: folder, language: Path.basename(from),
language: Path.basename(folder), files: files
files: Path.wildcard(Path.join(folder, "**"))
}} }}
end end
end end
defp translate_and_save(batch) do
target_languages = Languages.list() -- List.wrap(batch.language)
target_languages
|> Enum.each(fn language ->
language_path = Path.join([output_path, batch.batch, language])
File.mkdir_p(language_path)
batch.loaded
|> Enum.each(fn {filename, data} ->
{:ok, translation} = Translator.translate(data, language, batch.language)
file_path = Path.join(language_path, filename)
Parser.save(file_path, translation)
end)
end)
:ok
end
end end

View File

@ -55,9 +55,7 @@ defmodule Translator.Parser do
@spec availiable() :: availiable_parsers_map @spec availiable() :: availiable_parsers_map
defp availiable() do defp availiable() do
parsers = Application.get_env(:translator, :parsers) Application.get_env(:translator, :parsers)
parsers
|> Enum.reduce(%{}, fn parser, avaliable_extensions_map -> |> Enum.reduce(%{}, fn parser, avaliable_extensions_map ->
{:ok, extensions} = parser.extensions() {:ok, extensions} = parser.extensions()

View File

@ -7,19 +7,19 @@ defmodule Translator.Parser.JSON do
@behaviour Translator.Parser.Base @behaviour Translator.Parser.Base
@impl Translator.Parser.Base @impl true
@spec parse(contents) :: {:ok, data} | {:error, atom | Jason.DecodeError.t()} @spec parse(contents) :: {:ok, data} | {:error, atom | Jason.DecodeError.t()}
def parse(contents) do def parse(contents) do
Jason.decode(contents) Jason.decode(contents)
end end
@impl Translator.Parser.Base @impl true
@spec generate(data) :: {:ok, contents} | {:error, any} @spec generate(data) :: {:ok, contents} | {:error, any}
def generate(data) do def generate(data) do
Jason.encode(data) Jason.encode(data)
end end
@impl Translator.Parser.Base @impl true
@spec extensions() :: {:ok, extensions} @spec extensions() :: {:ok, extensions}
def extensions(), do: {:ok, @extensions} def extensions(), do: {:ok, @extensions}
end end

View File

@ -7,13 +7,13 @@ defmodule Translator.Parser.YAML do
@behaviour Translator.Parser.Base @behaviour Translator.Parser.Base
@impl Translator.Parser.Base @impl true
@spec parse(contents) :: {:ok, data} | {:error, atom | Jason.DecodeError.t()} @spec parse(contents) :: {:ok, data} | {:error, atom | Jason.DecodeError.t()}
def parse(contents) do def parse(contents) do
YamlElixir.read_from_string(contents) YamlElixir.read_from_string(contents)
end end
@impl Translator.Parser.Base @impl true
@spec generate(data) :: {:ok, contents} | {:error, any} @spec generate(data) :: {:ok, contents} | {:error, any}
def generate(data) do def generate(data) do
{:ok, to_yaml(data)} {:ok, to_yaml(data)}
@ -35,7 +35,7 @@ defmodule Translator.Parser.YAML do
defp value_to_yaml(value, key, indentation) when is_map(value), defp value_to_yaml(value, key, indentation) when is_map(value),
do: "#{indentation}#{key}:\n#{to_yaml(value, "#{indentation} ")}" do: "#{indentation}#{key}:\n#{to_yaml(value, "#{indentation} ")}"
@impl Translator.Parser.Base @impl true
@spec extensions() :: {:ok, extensions} @spec extensions() :: {:ok, extensions}
def extensions(), do: {:ok, @extensions} def extensions(), do: {:ok, @extensions}
end end

View File

@ -1,4 +1,61 @@
defmodule Translator do defmodule Translator do
defdelegate translate(text, to, from), to: YandexTranslate @type text :: String.t()
defdelegate translate(text, to), to: YandexTranslate @type locale :: String.t()
@type from :: locale
@type to :: locale
@type message :: String.t()
@type translator :: atom
@spec availiable :: [atom]
def availiable(), do: Application.get_env(:translator, :translators)
@spec default :: atom
def default(), do: availiable() |> List.first()
def translate(data, to, from, translator \\ default())
@spec translate(nil, to, from, translator) :: {:ok, nil} | {:error, message}
def translate(value, _to, _from, _translator)
when is_nil(value) or is_number(value) or is_boolean(value) or is_atom(value),
do: {:ok, value}
@spec translate(text, to, from, translator) :: {:ok, text} | {:error, message}
def translate(text, to, from, translator) when is_bitstring(text),
do: translator.translate(text, to, from)
@spec translate(map, to, from, translator) :: {:ok, map} | {:error, message}
def translate(map, to, from, translator) when is_map(map) do
{:ok, translate_map!(map, to, from, translator)}
end
@spec translate([map | text], to, from, translator) :: {:ok, [map | text]} | {:error, message}
def translate(list, to, from, translator) when is_list(list) do
{:ok, translate_list!(list, to, from, translator)}
end
@spec translate_list!(list, to, from, translator) :: [map | text] | {:error, message}
defp translate_list!(list, to, from, translator) when is_list(list) do
list
|> Enum.map(fn source ->
case translate(source, to, from, translator) do
{:ok, translation} -> translation
{:error, _message} -> source
end
end)
end
@spec translate_map!(map, to, from, translator) :: map | {:error, message}
defp translate_map!(data, to, from, translator) when is_map(data) do
data
|> Map.keys()
|> Enum.map(fn key ->
source = Map.fetch!(data, key)
case translate(source, to, from, translator) do
{:ok, translation} -> {key, translation}
{:error, _message} -> {key, source}
end
end)
|> Map.new()
end
end end

View File

@ -0,0 +1,22 @@
defmodule Translator.Base do
@typedoc """
Plain Text
"""
@type text :: String.t()
@typedoc """
Locale
"""
@type locale :: String.t()
@type from :: locale
@type to :: locale
@typedoc """
Error Message
"""
@type message :: String.t()
@callback detect(text) :: {:ok, locale} | {:error, message}
@callback translate(text, to) :: {:ok, text} | {:error, message}
@callback translate(text, to, from) :: {:ok, text} | {:error, message}
end

View File

@ -0,0 +1,57 @@
defmodule Translator.Yandex do
@type text :: String.t()
@type locale :: String.t()
@type from :: locale
@type to :: locale
@type message :: String.t()
@behaviour Translator.Base
@impl true
@spec detect(text) :: {:ok, locale} | {:error, message}
def detect(text) do
case YandexTranslate.detect(text) do
%{languageCode: locale} -> {:ok, locale}
%{message: message} -> {:error, message}
end
end
@impl true
@spec translate(text, to) :: {:ok, text} | {:error, message}
def translate(text, to) do
case detect(text) do
{:ok, from} -> translate(text, to, from)
{:error, message} -> {:error, message}
end
end
@impl true
@spec translate(text, to, from) :: {:ok, text} | {:error, message}
def translate(text, to, from) do
case YandexTranslate.translate(text, to, from) do
%{translations: translations} ->
translations
|> List.first()
|> Map.fetch(:text)
|> normalize()
%{message: message} ->
{:error, message}
end
end
@spec normalize({:ok, text}) :: {:ok, text} | {:error, message}
defp normalize({:ok, text}), do: normalize(text)
@spec normalize(text) :: {:ok, text} | {:error, message}
defp normalize(text) do
result =
text
|> String.replace(~r/(<)(\s)?(\/)?(\s)?(\w+)(\s)?(\/)?(\s)?(>)/, "\\1\\3\\5\\7\\9")
|> String.replace(~r/<(\w|\/)+>/, fn tag -> String.downcase(tag) end)
|> String.replace("> <", "><")
|> String.replace("% {", "%{")
{:ok, result}
end
end