Starting to work on #4 and #5

Focus_Locality/Sentence_Embedding_Approach/Testing/Profile.py

@@ -5,24 +5,25 @@
 """
 import SIFpreprocessing_test
 import SentenceExtraction_test
-from itertools import izip
+import sys
+sys.path.append("/home/ahalt/MITIE/")
+from mitie import named_entity_extractor
 
-
-def main(lang, NER, TextTest, word2vec_Dictionary, loaded_model):
+def process_story(lang, ner_model, NER, TextTest, word2vec_Dictionary, loaded_model):
 
 #### test
 #    lang = 'en'
 #    TextTest = '''105 words 4 July 2014 03:44 All Africa AFNWS English Mogadishu, Jul 04, 2014 (Tunis Afrique Presse/All Africa Global Media via COMTEX) -- A member of the Somali Federal Parliament has been shot dead by unknown gunmen on Thursday morning in Mogadishu, officials said. Ahmed Mohamud Hayd was killed in a drive-by shooting after he left his hotel in a heavily policed area, witnesses said. His bodyguard was also killed and a parliamentary secretary wounded in the shooting. Al-Shabab spokesman Abdulaziz Abu Musab said the group had carried out the "targeted assassination". At least five members of the Parliament have been shot since the beginning of the year.  '''
 #    NER = 'Mitie' #Stanford or Mitie or other
 
-    SentenceListTest, LocTest = SentenceExtraction_test.main(TextTest, NER, lang)
+    SentenceListTest, LocTest = SentenceExtraction_test.main(TextTest, ner_model, NER, lang)
     embTest = SIFpreprocessing_test.main(0.01, lang, TextTest, word2vec_Dictionary)
 
     predictedResult = loaded_model.predict(embTest)
 
     locDic = dict()
     resDic = dict()
-    for res, loc in izip(predictedResult, LocTest):
+    for res, loc in zip(predictedResult, LocTest):
         loc = loc.lower().strip()
         if res==1:
             if loc not in resDic:
@@ -38,4 +39,37 @@ def main(lang, NER, TextTest, word2vec_Dictionary, loaded_model):
     else: 
         sorted_loc = sorted(locDic, key=locDic.__getitem__, reverse=True)
 
-    return sorted_loc[0]
+    try:
+        return sorted_loc[0]
+    except IndexError:
+        return ""
+
+
+
+def main(lang, NER, test_docs, word2vec_Dictionary, loaded_model):
+    """
+    test_docs is now a list of documents
+    Note: didn't test for Stanford
+    """
+
+    print("loading NER model...")
+    #if NER == 'Stanford': 
+    #    # Stanford setup
+    #    os.environ['CLASSPATH'] = '../stanford/stanford-ner.jar'  
+    #    if lang == 'es':
+    #        st = StanfordNERTagger('../stanford/stanford-spanish-corenlp-models-current/edu/stanford/nlp/models/ner/spanish.ancora.distsim.s512.crf.ser.gz')
+    #    elif lang == 'en': 
+    #        st = StanfordNERTagger('../stanford/stanford-spanish-corenlp-models-current/edu/stanford/nlp/models/ner/spanish.ancora.distsim.s512.crf.ser.gz')
+    #    
+    #elif NER == 'Mitie':
+    #    sys.path.append('/home/ahalt/MITIE/mitielib')
+    #    if lang == 'es':
+    #        ner = named_entity_extractor('../MITIE-models/spanish/ner_model.dat')
+    #    elif lang == 'en': 
+    ner_model = named_entity_extractor('/home/ahalt/MITIE/MITIE-models/english/ner_model.dat')
+
+    locs = []
+    for doc in test_docs:
+        loc = process_story(lang, ner_model, NER, doc, word2vec_Dictionary, loaded_model)
+        locs.append(loc)
+    return locs

Focus_Locality/Sentence_Embedding_Approach/Testing/Readme.md

@@ -10,15 +10,15 @@ import Profile
 
 
 # define the news language 
-lang = en
+lang = 'en'
 NER = 'Mitie' #Stanford or Mitie or other
 
 if lang =='es': 
     word2vec_model = '/UserPath/wiki.es.vec'
     word2vec_Dictionary = FastVector(vector_file= word2vec_model)
     word2vec_Dictionary.apply_transform('./fastText_multilingual/alignment_matrices/es.txt')
 elif lang =='en':
-    word2vec_model = '/UserPath/wiki.en.vec'
+    word2vec_model = '/home/ahalt/wiki.en.vec'
     word2vec_Dictionary = FastVector(vector_file= word2vec_model)
     word2vec_Dictionary.apply_transform('./fastText_multilingual/alignment_matrices/en.txt')
 elif lang == 'ar':
@@ -30,11 +30,21 @@ elif lang == 'ar':
 model = "./model/model_"+lang+".sav"
 loaded_model = pickle.load(open(model, 'rb'))
 
+TextTest = ['A sentence about nowhere', 'After a week of calm in Boston, violence broke out in Amherst and Williamstown.']
 print( Profile.main(lang, NER, TextTest, word2vec_Dictionary, loaded_model) ) 
 
 ```
 
-You need to change NER path in ```SentenceExtraction_test.py``` if you want to work with Mitie or Stanford. 
+
+## Other information
+
+- You need to change NER path in ```SentenceExtraction_test.py``` if you want to work with Mitie or Stanford. 
+- You must use Python 2! The pickled model can only be loaded from Python 2. Loading with Python 3 gives a Unicode error.
+- You'll need to uncompress the English model in `models/` (currently only English is available)
+- Make sure to upgrade `sklearn` to 0.20
+- Make sure to download the nltk models, too:  
+    >>> import nltk
+    >>> nltk.download('punkt')
 
 ## Dependencies: 
 

Focus_Locality/Sentence_Embedding_Approach/Testing/SentenceExtraction_test.py

@@ -8,6 +8,7 @@
 from nltk.tag import StanfordNERTagger
 import sys, os
 import json
+sys.path.append("/home/ahalt/MITIE/mitielib")
 from mitie import *
 from polyglot.text import Text
 
@@ -17,7 +18,6 @@
 
 
 def get_JSONfile(JSONfileName):
-
     with open(JSONfileName) as f:
         content = json.load(f)
 
@@ -27,7 +27,6 @@ def get_JSONfile(JSONfileName):
 
 # polyglot function to extract location names
 def polyglotNER(textNews):
-
     locDic = {}
  #   print textNews
     text = Text(textNews)
@@ -47,9 +46,9 @@ def polyglotNER(textNews):
 
 
 # Mitie function to extract location names
-def MitieNER(tokens, ner):
+def MitieNER(tokens, ner_model):
 
-    entities = ner.extract_entities(tokens)
+    entities = ner_model.extract_entities(tokens)
     locDic = {}
 
     for e in entities:
@@ -105,38 +104,17 @@ def stanfordNER(text, st):
     return locDic
 
 
-def main(textTest, NER, lang):
-
-
+def main(textTest, ner_model, NER, lang):
     Sent = list()
     Loc = list()
 
-    print("loading NER model...")
-    if NER == 'Stanford': 
-        # Stanford setup
-        os.environ['CLASSPATH'] = '../stanford/stanford-ner.jar'  
-        if lang == 'es':
-            st = StanfordNERTagger('../stanford/stanford-spanish-corenlp-models-current/edu/stanford/nlp/models/ner/spanish.ancora.distsim.s512.crf.ser.gz')
-        elif lang == 'en': 
-            st = StanfordNERTagger('../stanford/stanford-spanish-corenlp-models-current/edu/stanford/nlp/models/ner/spanish.ancora.distsim.s512.crf.ser.gz')
-
-    elif NER == 'Mitie':
-        sys.path.append('../MITIE-master/mitielib')
-        if lang == 'es':
-            ner = named_entity_extractor('../MITIE-models/spanish/ner_model.dat')
-        elif lang == 'en': 
-            ner = named_entity_extractor('../MITIE-models/english/ner_model.dat')
-
-
-
     # load the country names in different languages
     if lang == 'en': 
         JSONfileName = './data/english_country_names.json'
     elif lang == 'es': 
         JSONfileName = './data/spanish_country_names.json'
     elif lang == 'ar': 
         JSONfileName = './data/arabic_country_names.json'
-
 
     try:
         countries = get_JSONfile(JSONfileName).keys()
@@ -153,7 +131,7 @@ def main(textTest, NER, lang):
         loc_stan= stanfordNER(text, st)
     elif NER == 'Mitie':   
         tokens = tokenize(str(text))
-        loc_stan= MitieNER(tokens, ner)
+        loc_stan= MitieNER(tokens, ner_model)
     else:
         loc_stan = polyglotNER(texttt)