From 92a54621f0d2ec0435f1a4366dec1a9c7fd3ab7e Mon Sep 17 00:00:00 2001
From: Jonathan Lundy <jonathan.lundy@rakuten.com>
Date: Fri, 26 Jun 2020 17:03:39 -0600
Subject: [PATCH] update rpsl whois

---
 utils/registry/dn42/rpsl/filedom.py   |  2 ++
 utils/registry/dn42/rpsl/rspldom.py   | 17 +++++++++++++----
 utils/registry/dn42/rpsl/schema.py    | 18 +++++++++++++-----
 utils/registry/rpsl_index/__init__.py |  7 +++----
 4 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/utils/registry/dn42/rpsl/filedom.py b/utils/registry/dn42/rpsl/filedom.py
index 899feb1f2..329bb0104 100644
--- a/utils/registry/dn42/rpsl/filedom.py
+++ b/utils/registry/dn42/rpsl/filedom.py
@@ -262,4 +262,6 @@ def index_files(path: str,
         for f in files:
             dom = FileDOM.from_file(os.path.join(root, f))
             dom.namespace = namespace
+            if dom.schema in primary_keys:
+                dom.primary_key = primary_keys[dom.schema]
             yield dom
diff --git a/utils/registry/dn42/rpsl/rspldom.py b/utils/registry/dn42/rpsl/rspldom.py
index 8a11d0648..3312ea431 100644
--- a/utils/registry/dn42/rpsl/rspldom.py
+++ b/utils/registry/dn42/rpsl/rspldom.py
@@ -19,7 +19,8 @@ class RPSL:
         self._config = config
         self._files = {}  # type: Dict[Tuple[str, str], str]
         self._lookup = {}  # type: Dict[str, List[Tuple[str, str]]]
-        self._links = {}  # type: Dict[Tuple[str, str], List[Tuple[str, str]]]
+        self._links = {}  \
+            # type: Dict[Tuple[str, str], List[Tuple[str, str, str]]]
         self._nettree = None  # type: NetTree
         self._schema = {}  # type: Dict[str, SchemaDOM]
         self._load_index()
@@ -36,8 +37,9 @@ class RPSL:
             for line in fd.readlines():
                 sp = line.strip().split(sep="|")
                 key = (sp[0], sp[1])
-                self._links[key] = self._lookup.get(key, [])
-                self._links[key].append((sp[2], sp[3]))
+                arr = self._links.get(key, [])
+                arr.append((sp[2], sp[3], sp[4]))
+                self._links[key] = arr
 
         self._nettree = NetTree.read_csv(self._config.nettree_file)
 
@@ -71,9 +73,16 @@ class RPSL:
         if schema is None:
             keys = self._lookup.get(text, [])
 
+        related = set()
+
         for i in keys:
             yield self.load_file(self._files[i])
-            print(self.links(i))
+            for link in self.links(i):
+                key = (link[1], link[2])
+                related.add(key)
+
+        for i in related:
+            yield self.load_file(self._files[i])
 
     def load_file(self, fn: str) -> FileDOM:
         "load file"
diff --git a/utils/registry/dn42/rpsl/schema.py b/utils/registry/dn42/rpsl/schema.py
index abe70290e..8b90a39d3 100644
--- a/utils/registry/dn42/rpsl/schema.py
+++ b/utils/registry/dn42/rpsl/schema.py
@@ -9,6 +9,7 @@ import log
 from .filedom import FileDOM, Row
 
 DOM = TypeVar("DOM", bound="FileDOM")
+STATE = TypeVar("STATE", bound="State")
 
 
 class Level(Enum):
@@ -34,6 +35,11 @@ class State:
     def __str__(self) -> str:
         return "PASS" if self.state else "FAIL"
 
+    def extend(self, state: STATE):
+        "apply state to state"
+        self.msgs.extend(state.msgs)
+        self.state = state.state
+
     def print_msgs(self):
         """print out state info"""
         for (level, row, msg) in self.msgs:
@@ -156,16 +162,18 @@ class SchemaDOM:
         if state is None:
             state = State()
 
+        file_state = State()
         if not f.valid:
-            state.error(Row("", "", 0, f.src), "file does not parse")
+            file_state.error(Row("", "", 0, f.src), "file does not parse")
 
-        state = self._check_file_structure(state, f)
-        state = self._check_file_values(state, f, lookups)
-        state = inetnum_check(state, f)
+        file_state = self._check_file_structure(file_state, f)
+        file_state = self._check_file_values(file_state, f, lookups)
+        file_state = inetnum_check(file_state, f)
 
         print("CHECK\t%-10s\t%-44s\t%s\tMNTNERS: %s" %
-              (f.schema, f.src.split("/")[-1], state, ','.join(f.mntner)))
+              (f.schema, f.src.split("/")[-1], file_state, ','.join(f.mntner)))
 
+        state.extend(file_state)
         return state
 
     def _check_file_structure(self, state: State, f: FileDOM) -> State:
diff --git a/utils/registry/rpsl_index/__init__.py b/utils/registry/rpsl_index/__init__.py
index 52a70893f..bd3eb352f 100644
--- a/utils/registry/rpsl_index/__init__.py
+++ b/utils/registry/rpsl_index/__init__.py
@@ -27,9 +27,8 @@ def run(args: List[str], env: Dict[str, str]) -> int:
         return 1
 
     config = Config.from_path(path)
-    if not os.path.exists(config.index_file) or \
-            not os.path.exists(config.schema_file):
-        print("RPSL index files not found. do `rpsl index`?", file=sys.stderr)
+    if not os.path.exists(config.config_file):
+        print("RPSL config files not found. do `rpsl init`?", file=sys.stderr)
         return 1
 
     if not os.path.isdir(config.schema_dir):
@@ -150,7 +149,7 @@ def generate_links(
     for (link, refs) in links.items():
         d = dom.get(link)
         if d is None:
-            return
+            continue
 
         found = False
         for ref in refs: