first commit

compiler/src/input.rs

@@ -0,0 +1,145 @@
+use std::collections::BTreeMap;
+use namegen::{SampleSet, Sample};
+use crate::manifest::SourceManifest;
+
+#[derive(Default)]
+pub struct SourceCollection {
+    sources: BTreeMap<String, Source>
+}
+
+impl SourceCollection {
+    pub fn sources(&self) -> impl Iterator<Item = (&str, &Source)> {
+        self.sources.iter().map(|(k, v)| (k.as_str(), v))
+    }
+
+    pub fn source(&self, name: &str) -> Option<&Source> {
+        self.sources.get(name)
+    }
+
+    pub fn source_mut(&mut self, name: &str) -> &mut Source {
+        if !self.sources.contains_key(name) {
+            self.sources.insert(name.to_owned(), Source::default());
+        }
+
+        self.sources.get_mut(name).unwrap()
+    }
+
+    pub fn load_full_names(&mut self, data: &str, source: &SourceManifest) -> Result<(), &'static str> {
+        for line in data.lines() {
+            if line.len() < 1 {
+                continue
+            }
+
+            if let Some(prefix) = &source.comment_prefix {
+                if line.starts_with(prefix) {
+                    continue;
+                }
+            }
+
+            for (i, token) in line.split(' ').enumerate() {
+                if let Some(empty_token) = &source.empty_token {
+                    if token == empty_token.as_str() {
+                        continue;
+                    }
+                }
+
+                let mut target = self.source_mut(&source.targets[i]);
+
+                let mut token = if source.to_lower {
+                    token.to_owned().to_lowercase()
+                } else {
+                    token.to_owned()
+                };
+
+                target.set_mut("list").add_sample(Sample::Word(token))
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn load_labeled_groups(&mut self, data: &str, source: &SourceManifest) -> Result<(), &'static str> {
+        let mut target_index = 0usize;
+        let mut new_group = true;
+        let mut current_key = String::new();
+
+        for line in data.lines() {
+            if line.len() < 1 {
+                new_group = true;
+                continue;
+            }
+
+            if let Some(separator) = &source.separator {
+                if line.starts_with(separator) {
+                    target_index += 1;
+                    new_group;
+                    continue;
+                }
+            }
+
+            if let Some(prefix) = &source.comment_prefix {
+                if line.starts_with(prefix) {
+                    continue;
+                }
+            }
+
+            let mut target = self.source_mut(&source.targets[target_index]);
+
+            if new_group {
+                current_key = line.to_owned();
+                let labels: Vec<String> = line.split(' ').map(|s| s.to_owned()).collect();
+
+                target.set_labeled_mut(line, labels.as_slice());
+                new_group = false;
+
+                continue;
+            }
+
+            let line = if source.to_lower {
+                line.to_owned().to_lowercase()
+            } else {
+                line.to_owned()
+            };
+
+            let tokens: Vec<String> = line.split(' ').map(|s| s.to_owned()).collect();
+            let set = target.set_mut(current_key.as_str());
+
+            if tokens.len() != set.labels().len() {
+                return Err("Token length != label count");
+            }
+
+            set.add_sample(
+                Sample::Tokens(tokens)
+            );
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Default)]
+pub struct Source {
+    sets: BTreeMap<String, SampleSet>
+}
+
+impl Source {
+    pub fn sets(&self) -> impl Iterator<Item = &SampleSet> {
+        self.sets.values()
+    }
+
+    pub fn set_mut(&mut self, name: &str) -> &mut SampleSet {
+        if !self.sets.contains_key(name) {
+            self.sets.insert(name.to_owned(), SampleSet::new());
+        }
+
+        self.sets.get_mut(name).unwrap()
+    }
+
+    pub fn set_labeled_mut<S: AsRef<str>>(&mut self, name: &str, labels: &[S]) -> &mut SampleSet {
+        if !self.sets.contains_key(name) {
+            self.sets.insert(name.to_owned(), SampleSet::with_labels(labels));
+        }
+
+        self.sets.get_mut(name).unwrap()
+    }
+}

compiler/src/lib.rs

@@ -0,0 +1,6 @@
+#[macro_use]
+extern crate serde_derive;
+
+pub mod manifest;
+pub mod input;
+pub mod output;

compiler/src/main.rs

@@ -0,0 +1,172 @@
+use std::process;
+use std::env;
+use std::fs::{read_dir, File};
+use std::collections::HashMap;
+use namegen_compiler::manifest::Manifest;
+use std::env::join_paths;
+use std::path::Path;
+use namegen_compiler::input::SourceCollection;
+use glob::glob;
+use std::io::{Read};
+use namegen::{Name, NamePart};
+use namegen_compiler::output::{Output, Collection, CollectionItem};
+
+const COMMON_FORMAT_NAMES: &'static [&str] = &[
+    "full_name", "full_name:female", "full_name:male",
+];
+
+fn main() {
+    let source_dir = match env::var("SOURCE_DIR") {
+        Ok(v) => v,
+        Err(_) => {
+            eprintln!("SOURCE_DIR missing");
+            process::exit(1);
+        }
+    };
+
+    let dest_dir = match env::var("DESTINATION_DIR") {
+        Ok(v) => v,
+        Err(_) => {
+            eprintln!("DESTINATION_DIR missing");
+            process::exit(1);
+        }
+    };
+
+    let mut data = String::with_capacity(2048);
+    let mut collection = Collection{items: Vec::new()};
+
+    for entry in read_dir(&source_dir).unwrap() {
+        let entry = entry.unwrap();
+        if !entry.file_type().unwrap().is_file() {
+            continue;
+        }
+
+        let path = Path::new(&source_dir).join(entry.file_name());
+        eprintln!("Loading manifest {}", path.to_str().unwrap());
+        let manifest: Manifest = serde_yaml::from_reader(File::open(path.clone()).unwrap()).unwrap();
+
+        let mut coll = SourceCollection::default();
+        for source in manifest.sources.iter() {
+            let path = path.with_file_name(&source.files);
+
+            for entry in glob(path.to_str().unwrap()).expect("Failed to read glob pattern") {
+                let path = entry.expect("Invalid file entry.");
+
+                data.clear();
+                File::open(path.clone()).unwrap().read_to_string(&mut data);
+
+                eprintln!("Loading source {} (size: {})", path.clone().to_str().unwrap(), data.len());
+
+                match source.kind.as_str() {
+                    "full_names" => { coll.load_full_names(&data, source).unwrap() }
+                    "labeled_groups" => { coll.load_labeled_groups(&data, source).unwrap() }
+                    _ => {
+                        eprintln!("Unknown source file kind {}", source.kind);
+                        process::exit(1);
+                    }
+                }
+            }
+        }
+
+        let mut name = Name::new();
+
+        for part in manifest.parts.iter() {
+            eprintln!("Building part {}...", &part.name);
+
+            let source = coll.source(&part.source).expect("Source not found.");
+            let mut part = match part.kind.as_str() {
+                "cfgrammar" => NamePart::new_cfgrammar(
+                    &part.name, &part.format_rules, &part.initial_tokens,
+                    part.rules.rlf, part.rules.ral,
+                ),
+                "markov" => NamePart::new_markov(
+                    &part.name, &part.format_rules, &part.initial_tokens,
+                    part.rules.lrs, part.rules.lrm, part.rules.lre, part.rules.rlf,
+                ),
+                "wordlist" => NamePart::new_wordlist(
+                    &part.name, &part.format_rules
+                ),
+
+                _ => {
+                    eprintln!("Unknown part kind {}", &part.kind);
+                    process::exit(1);
+                },
+            };
+
+            for set in source.sets() {
+                if set.labels().len() == 0 {
+                    eprintln!("\tSample list: {} samples", set.samples().len());
+                } else {
+                    eprintln!("\tSample group: {}", set.labels().join(" "));
+                }
+                part.learn(set).unwrap()
+            }
+
+            name.add_part(part);
+        }
+
+        for format in manifest.formats.iter() {
+            name.add_format(&format.name, &format.template)
+        }
+
+        let mut examples = HashMap::new();
+        for format in name.formats() {
+            if let Some(gen) = name.generate(format.name()) {
+		examples.insert(format.name().to_owned(), Vec::with_capacity(40));
+
+                for (i, result) in gen.enumerate().take(40) {
+                   examples.get_mut(format.name()).unwrap().push(result);
+                }
+            }
+        }
+
+        eprint!("\n");
+        eprintln!("Sample output");
+        for format_name in COMMON_FORMAT_NAMES.iter() {
+            if let Some(gen) = name.generate(format_name) {
+                eprint!("\n    ");
+
+                for (i, result) in gen.enumerate().take(72) {
+                    eprint!("{result:<width$} ", result = result, width = 19);
+                    if i > 0 && (i % 4 == 3) {
+                        eprint!("\n    ");
+                    }
+                }
+
+                eprint!("\n");
+            }
+        }
+
+        if let Some(gen) = name.generate("long_full_name:male") {
+            eprint!("\n    ");
+
+            for (i, result) in gen.enumerate().take(16) {
+                eprint!("{result:<width$} ", result = result, width = 52);
+                if i > 0 && (i % 2 == 1) {
+                    eprint!("\n    ");
+                }
+            }
+
+            eprint!("\n");
+        }
+
+        let path = Path::new(&dest_dir).join(format!("{}.json", &manifest.name));
+        let mut output = Output{
+            name: manifest.name.clone(),
+            metadata: manifest.metadata.clone(),
+            data: name,
+            examples: examples,
+        };
+
+        eprintln!("Writing {}", path.clone().to_str().unwrap());
+
+        serde_json::to_writer(File::create(path).expect("Opening output filed failed"), &output).expect("Writing output failed.");
+
+        collection.items.push(CollectionItem{name: output.name.clone(), metadata: output.metadata.clone()});
+
+        eprint!("\n\n");
+    }
+
+    let path = Path::new(&dest_dir).join("_collection.json");
+    serde_json::to_writer(File::create(path).expect("Opening output filed failed"), &collection).expect("Writing collection failed.");
+}

compiler/src/manifest.rs

@@ -0,0 +1,60 @@
+use std::collections::HashMap;
+use namegen::FormattingRule;
+
+#[derive(Deserialize)]
+pub struct Manifest {
+    pub name: String,
+    #[serde(default)]
+    pub metadata: HashMap<String, String>,
+    pub sources: Vec<SourceManifest>,
+    pub formats: Vec<FormatManifest>,
+    pub parts: Vec<PartManifest>,
+}
+
+#[derive(Deserialize)]
+pub struct PartManifest {
+    pub name: String,
+    pub kind: String,
+    #[serde(default)]
+    pub initial_tokens: Vec<String>,
+    #[serde(default)]
+    pub rules: PartManifestRule,
+    pub source: String,
+    #[serde(default)]
+    pub format_rules: Vec<FormattingRule>,
+}
+
+#[derive(Deserialize, Default)]
+pub struct PartManifestRule {
+    #[serde(default)]
+    pub rlf: bool,
+    #[serde(default)]
+    pub ral: bool,
+    #[serde(default)]
+    pub lrs: bool,
+    #[serde(default)]
+    pub lrm: bool,
+    #[serde(default)]
+    pub lre: bool,
+}
+
+#[derive(Deserialize)]
+pub struct SourceManifest {
+    pub files: String,
+    pub kind: String,
+    pub targets: Vec<String>,
+    #[serde(default)]
+    pub to_lower: bool,
+    #[serde(default)]
+    pub empty_token: Option<String>,
+    #[serde(default)]
+    pub separator: Option<String>,
+    #[serde(default)]
+    pub comment_prefix: Option<String>,
+}
+
+#[derive(Deserialize)]
+pub struct FormatManifest {
+    pub name: String,
+    pub template: String,
+}

compiler/src/output.rs

@@ -0,0 +1,24 @@
+use std::collections::HashMap;
+use namegen::Name;
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Output {
+    pub name: String,
+    pub metadata: HashMap<String, String>,
+    pub data: Name,
+    pub examples: HashMap<String, Vec<String>>,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Collection {
+    pub items: Vec<CollectionItem>,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CollectionItem {
+    pub name: String,
+    pub metadata: HashMap<String, String>,
+}