Minimal worked solution
This commit is contained in:
parent
b45a818fa4
commit
2899e4281e
@ -6,6 +6,10 @@ config :logger, :console,
|
|||||||
metadata: [:request_id]
|
metadata: [:request_id]
|
||||||
|
|
||||||
config :translator,
|
config :translator,
|
||||||
|
input_folder: "input",
|
||||||
|
output_folder: "output",
|
||||||
|
base_folder: "priv",
|
||||||
|
translators: [Translator.Yandex],
|
||||||
parsers: [Translator.Parser.YAML, Translator.Parser.JSON],
|
parsers: [Translator.Parser.YAML, Translator.Parser.JSON],
|
||||||
availiable_languages: ~w(en zh ar es hi de ru be uk kk bn pl pt it fr)
|
availiable_languages: ~w(en zh ar es hi de ru be uk kk bn pl pt it fr)
|
||||||
|
|
||||||
|
@ -1,28 +1,79 @@
|
|||||||
defmodule Translator.Batcher do
|
defmodule Translator.Batcher do
|
||||||
alias Translator.Languages
|
alias Translator.{Languages, Parser}
|
||||||
|
|
||||||
|
def process(batch) do
|
||||||
|
# ,:ok <- translate_and_save(loaded)
|
||||||
|
with {:ok, formed} <- parse(batch),
|
||||||
|
{:ok, loaded} <- load(formed),
|
||||||
|
:ok <- translate_and_save(loaded) do
|
||||||
|
loaded
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp base_folder(), do: Application.get_env(:translator, :base_folder, "priv")
|
||||||
|
defp output_folder(), do: Application.get_env(:translator, :output_folder, "output")
|
||||||
|
defp input_folder(), do: Application.get_env(:translator, :input_folder, "input")
|
||||||
|
|
||||||
|
defp input_path(), do: Path.join(base_folder(), input_folder())
|
||||||
|
defp output_path(), do: Path.join(base_folder(), output_folder())
|
||||||
|
|
||||||
|
defp load(batch) do
|
||||||
|
loaded =
|
||||||
|
batch.files
|
||||||
|
|> Enum.reduce([], fn filename, acc ->
|
||||||
|
path = Path.join(batch.from, filename)
|
||||||
|
{:ok, content} = Parser.load(path)
|
||||||
|
|
||||||
|
[{filename, content} | acc]
|
||||||
|
end)
|
||||||
|
|> Map.new()
|
||||||
|
|
||||||
|
{:ok, Map.put(batch, :loaded, loaded)}
|
||||||
|
end
|
||||||
|
|
||||||
@spec parse(String.t()) :: {:ok, map} | {:error, String.t()}
|
@spec parse(String.t()) :: {:ok, map} | {:error, String.t()}
|
||||||
def parse(path) do
|
defp parse(batch) do
|
||||||
batch = Path.basename(path)
|
from =
|
||||||
|
Path.join([input_path(), batch, Languages.pattern()])
|
||||||
folder =
|
|
||||||
Path.join(path, Languages.pattern())
|
|
||||||
|> Path.wildcard()
|
|> Path.wildcard()
|
||||||
|> List.first()
|
|> List.first()
|
||||||
|
|
||||||
case folder do
|
case from do
|
||||||
nil ->
|
nil ->
|
||||||
{:error, "Couldn't find input folder in batch '#{path}'"}
|
{:error, "Couldn't find input folder in batch at '#{from}'"}
|
||||||
|
|
||||||
_ ->
|
_ ->
|
||||||
|
files =
|
||||||
|
Path.wildcard(Path.join(from, "**"))
|
||||||
|
|> Enum.map(fn path -> Path.relative_to(path, from) end)
|
||||||
|
|
||||||
{:ok,
|
{:ok,
|
||||||
%{
|
%{
|
||||||
name: batch,
|
batch: batch,
|
||||||
folder: path,
|
from: from,
|
||||||
input: folder,
|
language: Path.basename(from),
|
||||||
language: Path.basename(folder),
|
files: files
|
||||||
files: Path.wildcard(Path.join(folder, "**"))
|
|
||||||
}}
|
}}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp translate_and_save(batch) do
|
||||||
|
target_languages = Languages.list() -- List.wrap(batch.language)
|
||||||
|
|
||||||
|
target_languages
|
||||||
|
|> Enum.each(fn language ->
|
||||||
|
language_path = Path.join([output_path, batch.batch, language])
|
||||||
|
|
||||||
|
File.mkdir_p(language_path)
|
||||||
|
|
||||||
|
batch.loaded
|
||||||
|
|> Enum.each(fn {filename, data} ->
|
||||||
|
{:ok, translation} = Translator.translate(data, language, batch.language)
|
||||||
|
file_path = Path.join(language_path, filename)
|
||||||
|
Parser.save(file_path, translation)
|
||||||
|
end)
|
||||||
|
end)
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
@ -55,9 +55,7 @@ defmodule Translator.Parser do
|
|||||||
|
|
||||||
@spec availiable() :: availiable_parsers_map
|
@spec availiable() :: availiable_parsers_map
|
||||||
defp availiable() do
|
defp availiable() do
|
||||||
parsers = Application.get_env(:translator, :parsers)
|
Application.get_env(:translator, :parsers)
|
||||||
|
|
||||||
parsers
|
|
||||||
|> Enum.reduce(%{}, fn parser, avaliable_extensions_map ->
|
|> Enum.reduce(%{}, fn parser, avaliable_extensions_map ->
|
||||||
{:ok, extensions} = parser.extensions()
|
{:ok, extensions} = parser.extensions()
|
||||||
|
|
||||||
|
@ -7,19 +7,19 @@ defmodule Translator.Parser.JSON do
|
|||||||
|
|
||||||
@behaviour Translator.Parser.Base
|
@behaviour Translator.Parser.Base
|
||||||
|
|
||||||
@impl Translator.Parser.Base
|
@impl true
|
||||||
@spec parse(contents) :: {:ok, data} | {:error, atom | Jason.DecodeError.t()}
|
@spec parse(contents) :: {:ok, data} | {:error, atom | Jason.DecodeError.t()}
|
||||||
def parse(contents) do
|
def parse(contents) do
|
||||||
Jason.decode(contents)
|
Jason.decode(contents)
|
||||||
end
|
end
|
||||||
|
|
||||||
@impl Translator.Parser.Base
|
@impl true
|
||||||
@spec generate(data) :: {:ok, contents} | {:error, any}
|
@spec generate(data) :: {:ok, contents} | {:error, any}
|
||||||
def generate(data) do
|
def generate(data) do
|
||||||
Jason.encode(data)
|
Jason.encode(data)
|
||||||
end
|
end
|
||||||
|
|
||||||
@impl Translator.Parser.Base
|
@impl true
|
||||||
@spec extensions() :: {:ok, extensions}
|
@spec extensions() :: {:ok, extensions}
|
||||||
def extensions(), do: {:ok, @extensions}
|
def extensions(), do: {:ok, @extensions}
|
||||||
end
|
end
|
||||||
|
@ -7,13 +7,13 @@ defmodule Translator.Parser.YAML do
|
|||||||
|
|
||||||
@behaviour Translator.Parser.Base
|
@behaviour Translator.Parser.Base
|
||||||
|
|
||||||
@impl Translator.Parser.Base
|
@impl true
|
||||||
@spec parse(contents) :: {:ok, data} | {:error, atom | Jason.DecodeError.t()}
|
@spec parse(contents) :: {:ok, data} | {:error, atom | Jason.DecodeError.t()}
|
||||||
def parse(contents) do
|
def parse(contents) do
|
||||||
YamlElixir.read_from_string(contents)
|
YamlElixir.read_from_string(contents)
|
||||||
end
|
end
|
||||||
|
|
||||||
@impl Translator.Parser.Base
|
@impl true
|
||||||
@spec generate(data) :: {:ok, contents} | {:error, any}
|
@spec generate(data) :: {:ok, contents} | {:error, any}
|
||||||
def generate(data) do
|
def generate(data) do
|
||||||
{:ok, to_yaml(data)}
|
{:ok, to_yaml(data)}
|
||||||
@ -35,7 +35,7 @@ defmodule Translator.Parser.YAML do
|
|||||||
defp value_to_yaml(value, key, indentation) when is_map(value),
|
defp value_to_yaml(value, key, indentation) when is_map(value),
|
||||||
do: "#{indentation}#{key}:\n#{to_yaml(value, "#{indentation} ")}"
|
do: "#{indentation}#{key}:\n#{to_yaml(value, "#{indentation} ")}"
|
||||||
|
|
||||||
@impl Translator.Parser.Base
|
@impl true
|
||||||
@spec extensions() :: {:ok, extensions}
|
@spec extensions() :: {:ok, extensions}
|
||||||
def extensions(), do: {:ok, @extensions}
|
def extensions(), do: {:ok, @extensions}
|
||||||
end
|
end
|
||||||
|
@ -1,4 +1,61 @@
|
|||||||
defmodule Translator do
|
defmodule Translator do
|
||||||
defdelegate translate(text, to, from), to: YandexTranslate
|
@type text :: String.t()
|
||||||
defdelegate translate(text, to), to: YandexTranslate
|
@type locale :: String.t()
|
||||||
|
@type from :: locale
|
||||||
|
@type to :: locale
|
||||||
|
@type message :: String.t()
|
||||||
|
@type translator :: atom
|
||||||
|
|
||||||
|
@spec availiable :: [atom]
|
||||||
|
def availiable(), do: Application.get_env(:translator, :translators)
|
||||||
|
|
||||||
|
@spec default :: atom
|
||||||
|
def default(), do: availiable() |> List.first()
|
||||||
|
|
||||||
|
def translate(data, to, from, translator \\ default())
|
||||||
|
|
||||||
|
@spec translate(nil, to, from, translator) :: {:ok, nil} | {:error, message}
|
||||||
|
def translate(value, _to, _from, _translator)
|
||||||
|
when is_nil(value) or is_number(value) or is_boolean(value) or is_atom(value),
|
||||||
|
do: {:ok, value}
|
||||||
|
|
||||||
|
@spec translate(text, to, from, translator) :: {:ok, text} | {:error, message}
|
||||||
|
def translate(text, to, from, translator) when is_bitstring(text),
|
||||||
|
do: translator.translate(text, to, from)
|
||||||
|
|
||||||
|
@spec translate(map, to, from, translator) :: {:ok, map} | {:error, message}
|
||||||
|
def translate(map, to, from, translator) when is_map(map) do
|
||||||
|
{:ok, translate_map!(map, to, from, translator)}
|
||||||
|
end
|
||||||
|
|
||||||
|
@spec translate([map | text], to, from, translator) :: {:ok, [map | text]} | {:error, message}
|
||||||
|
def translate(list, to, from, translator) when is_list(list) do
|
||||||
|
{:ok, translate_list!(list, to, from, translator)}
|
||||||
|
end
|
||||||
|
|
||||||
|
@spec translate_list!(list, to, from, translator) :: [map | text] | {:error, message}
|
||||||
|
defp translate_list!(list, to, from, translator) when is_list(list) do
|
||||||
|
list
|
||||||
|
|> Enum.map(fn source ->
|
||||||
|
case translate(source, to, from, translator) do
|
||||||
|
{:ok, translation} -> translation
|
||||||
|
{:error, _message} -> source
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
@spec translate_map!(map, to, from, translator) :: map | {:error, message}
|
||||||
|
defp translate_map!(data, to, from, translator) when is_map(data) do
|
||||||
|
data
|
||||||
|
|> Map.keys()
|
||||||
|
|> Enum.map(fn key ->
|
||||||
|
source = Map.fetch!(data, key)
|
||||||
|
|
||||||
|
case translate(source, to, from, translator) do
|
||||||
|
{:ok, translation} -> {key, translation}
|
||||||
|
{:error, _message} -> {key, source}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|> Map.new()
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
22
lib/translator/translator/base.ex
Normal file
22
lib/translator/translator/base.ex
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
defmodule Translator.Base do
|
||||||
|
@typedoc """
|
||||||
|
Plain Text
|
||||||
|
"""
|
||||||
|
@type text :: String.t()
|
||||||
|
|
||||||
|
@typedoc """
|
||||||
|
Locale
|
||||||
|
"""
|
||||||
|
@type locale :: String.t()
|
||||||
|
@type from :: locale
|
||||||
|
@type to :: locale
|
||||||
|
|
||||||
|
@typedoc """
|
||||||
|
Error Message
|
||||||
|
"""
|
||||||
|
@type message :: String.t()
|
||||||
|
|
||||||
|
@callback detect(text) :: {:ok, locale} | {:error, message}
|
||||||
|
@callback translate(text, to) :: {:ok, text} | {:error, message}
|
||||||
|
@callback translate(text, to, from) :: {:ok, text} | {:error, message}
|
||||||
|
end
|
57
lib/translator/translator/yandex.ex
Normal file
57
lib/translator/translator/yandex.ex
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
defmodule Translator.Yandex do
|
||||||
|
@type text :: String.t()
|
||||||
|
@type locale :: String.t()
|
||||||
|
@type from :: locale
|
||||||
|
@type to :: locale
|
||||||
|
@type message :: String.t()
|
||||||
|
|
||||||
|
@behaviour Translator.Base
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
@spec detect(text) :: {:ok, locale} | {:error, message}
|
||||||
|
def detect(text) do
|
||||||
|
case YandexTranslate.detect(text) do
|
||||||
|
%{languageCode: locale} -> {:ok, locale}
|
||||||
|
%{message: message} -> {:error, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
@spec translate(text, to) :: {:ok, text} | {:error, message}
|
||||||
|
def translate(text, to) do
|
||||||
|
case detect(text) do
|
||||||
|
{:ok, from} -> translate(text, to, from)
|
||||||
|
{:error, message} -> {:error, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
@spec translate(text, to, from) :: {:ok, text} | {:error, message}
|
||||||
|
def translate(text, to, from) do
|
||||||
|
case YandexTranslate.translate(text, to, from) do
|
||||||
|
%{translations: translations} ->
|
||||||
|
translations
|
||||||
|
|> List.first()
|
||||||
|
|> Map.fetch(:text)
|
||||||
|
|> normalize()
|
||||||
|
|
||||||
|
%{message: message} ->
|
||||||
|
{:error, message}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@spec normalize({:ok, text}) :: {:ok, text} | {:error, message}
|
||||||
|
defp normalize({:ok, text}), do: normalize(text)
|
||||||
|
|
||||||
|
@spec normalize(text) :: {:ok, text} | {:error, message}
|
||||||
|
defp normalize(text) do
|
||||||
|
result =
|
||||||
|
text
|
||||||
|
|> String.replace(~r/(<)(\s)?(\/)?(\s)?(\w+)(\s)?(\/)?(\s)?(>)/, "\\1\\3\\5\\7\\9")
|
||||||
|
|> String.replace(~r/<(\w|\/)+>/, fn tag -> String.downcase(tag) end)
|
||||||
|
|> String.replace("> <", "><")
|
||||||
|
|> String.replace("% {", "%{")
|
||||||
|
|
||||||
|
{:ok, result}
|
||||||
|
end
|
||||||
|
end
|
Loading…
x
Reference in New Issue
Block a user