Add slot filtering by intent to attempt to increase slot accuracy

.gitignore

@@ -124,3 +124,6 @@ glove/
 venv/
 env/
 glove.6B.zip
+.vscode
+.DS_Store
+*.pt

datasets/#count.py#

@@ -1,10 +0,0 @@
-filename = 'atis/train/seq.out'
-
-blah = set()
-with open(filename) as f:
-    for line in f:
-        line = line.rstrip().split()
-        for word in line:
-            blah.add(word)
-
-print(len(blah))

datasets/snips/train/label

@@ -13082,3 +13082,5 @@ PlayMusic
 RateBook
 SearchCreativeWork
 RateBook
+GetWeather
+GetWeather

datasets/snips/train/seq.in

@@ -13082,3 +13082,5 @@ play funky heavy bluesy
 rate the current album 2 points out of 6
 go to the photograph the inflated tear
 rate richard carvel 4 out of 6
+how does the weather look in york pa right now
+what s the weather like in new market

datasets/snips/train/seq.out

@@ -13082,3 +13082,5 @@ O B-playlist I-playlist I-playlist
 O O B-object_select B-object_type B-rating_value B-rating_unit O O B-best_rating 
 O O O B-object_type B-object_name I-object_name I-object_name 
 O B-object_name I-object_name B-rating_value O O B-best_rating 
+O O O O O O B-city B-state O B-timeRange
+O O O O O O B-city I-city 

dump_pickles.py

@@ -0,0 +1,15 @@
+from dataset import Corpus, build_glove
+import pickle
+
+train_corpus = Corpus('snips', 'train', 50)
+
+pickle.dump(train_corpus.word2idx, open( "word2idx.pkl", "wb" ))
+
+wordvecs = build_glove(train_corpus.word2idx, train_corpus.idx2word)
+pickle.dump(wordvecs, open( "wordvecs.pkl", "wb" ))
+
+slots = [slot for slot in train_corpus.slot2idx]
+pickle.dump(slots, open( "slots.pkl", "wb" ))
+
+intent = [intent for intent in train_corpus.intent2idx]
+pickle.dump(intent, open( "intents.pkl", "wb" ))

intent_filters.py

@@ -0,0 +1,46 @@
+import pickle
+# zz = torch.stack([z.repeat(50,1).transpose(0,1)])
+
+# GetWeather
+getWeather = ['state', 'city', 'timeRange', 'geographic_poi', 'country', 'facility']
+
+# PlayMusic
+playMusic = ['artist', 'album', 'playlist', 'service', 'rating_value', 'best_rating', 'music_item', 'track', 'playlist_owner', 'year', 'genre', 'rating_unit']
+
+# AddToPlaylist
+addToPlaylist = ['artist', 'album', 'playlist', 'service', 'rating_value', 'best_rating', 'music_item', 'track', 'playlist_owner', 'year', 'genre', 'rating_unit']
+
+# RateBook
+## I don't care
+
+# SearchScreeningEvent
+## I don't care
+
+# BookRestaurant
+## I don't care
+
+# SearchCreativeWork
+## I don't care
+
+generic = ['O', 'B-entity_name', 'B-object_name', 'B-object_type', 'I-entity_name', 'I-object_name', 'I-object_type']
+
+getWeatherOut = generic[:]
+for i in getWeather:
+    getWeatherOut.append('B-{}'.format(i))
+    getWeatherOut.append('I-{}'.format(i))
+
+playMusicOut = generic[:]
+for i in playMusic:
+    playMusicOut.append('B-{}'.format(i))
+    playMusicOut.append('I-{}'.format(i))
+
+addToPlaylistOut = generic[:]
+for i in addToPlaylist:
+    addToPlaylistOut.append('B-{}'.format(i))
+    addToPlaylistOut.append('I-{}'.format(i))
+
+out = {"GetWeather": getWeatherOut,
+       "PlayMusic": playMusicOut,
+       "AddToPlaylist": addToPlaylist}
+
+pickle.dump(out, open("slot_filters.pkl", "wb"))

requirements.txt

@@ -1,4 +1,4 @@
-torch
+torch==1.5.1
 seqeval
 scikit-learn
 transformers

test_query.py

@@ -17,10 +17,11 @@ EOS = ""
 word2idx = pickle.load(open("word2idx.pkl", "rb"))
 wordvecs = pickle.load(open("wordvecs.pkl", "rb"))
 slots = pickle.load(open("slots.pkl", "rb"))
+slot_filters = pickle.load(open("slot_filters.pkl", "rb"))
 intents = pickle.load(open("intents.pkl", "rb"))
 num_words = len(word2idx)
-num_intent = 7
-num_slot = 72
+num_intent = len(intents)
+num_slot = len(slots)
 filter_count = 300
 dropout = 0
 embedding_dim = 100
@@ -33,63 +34,71 @@ def pad_query(sequence):
         return sequence
 
 
-query = "What's the weather like in York PA right now?"
-q = query.lower().replace("'", " ").replace("?", " ").strip()
-# true_length = [len(q.split()), 0, 0, 0, 0, 0, 0 ,0]
-true_length = [len(q.split())]
-qq = torch.from_numpy(pad_query([word2idx[word] for word in q.split()]))
-    
-model = models.CNNJoint(num_words, embedding_dim, num_intent, num_slot, (filter_count,), 5, dropout, wordvecs)
-model.eval()
-
-model.load_state_dict(torch.load('snips_joint', map_location=torch.device('cpu')))
-criterion = torch.nn.CrossEntropyLoss(ignore_index=-1)
-
-pad_tensor = torch.from_numpy(pad_query([word2idx[w] for w in []]))
-# batch = torch.stack([qq, pad_tensor, pad_tensor, pad_tensor, pad_tensor, pad_tensor, pad_tensor, pad_tensor])    
-batch = torch.stack([qq])    
-
-pred_intent, pred_slots = model(batch)
-
-slt = [str(item) for batch_num, sublist in enumerate(pred_slots.max(1)[1].tolist()) for item in sublist[1:true_length[batch_num] + 1]]                                                                                                                      
-out_slots = [slots[int(c)] for c in slt]
-
-itnt = pred_intent.max(1)[1].tolist()[0]
-out_intent = intents[itnt]
-
-print("Input: {}\nIntent: {}\nSlots: {}".format(query, out_intent, out_slots))
-
-print("--- %s seconds ---" % (time.time() - start_time))
-
-# Write to output file
-out = ""
-collected_slots = {}
-active_slot_words = []
-active_slot_name = None
-for words, slot_preds, intent_pred in zip([q.split()], [out_slots], [out_intent]):
-    line = ""
-    for word, pred in zip(words, slot_preds):
-        line = line + word + " "
-        if pred == 'O':
-            if active_slot_name:
-                collected_slots[active_slot_name] = " ".join(active_slot_words)
-                active_slot_words = []
-                active_slot_name = None
-        else:
-            # Naive BIO handling: treat B- and I- the same...
-            new_slot_name = pred[2:]
-            if active_slot_name is None:
-                active_slot_words.append(word)
-                active_slot_name = new_slot_name
-            elif new_slot_name == active_slot_name:
-                active_slot_words.append(word)
+def predict(query):
+    q = query.lower().replace("'", " ").replace("?", " ").strip()
+    true_length = [len(q.split())]
+    qq = torch.from_numpy(pad_query([word2idx[word] if word in word2idx else word2idx["<pad>"] for word in q.split()]))
+        
+    model = models.CNNJoint(num_words, embedding_dim, num_intent, num_slot, (filter_count,), 5, dropout, wordvecs)
+    model.eval()
+
+    model.load_state_dict(torch.load('snips_joint', map_location=torch.device('cpu')))
+
+    batch = torch.stack([qq])    
+    pred_intent, pred_slots = model(batch)
+
+    itnt = pred_intent.max(1)[1].tolist()[0]
+    out_intent = intents[itnt]
+
+    if out_intent in slot_filters:
+        b = [1 if x in slot_filters[out_intent] else 0 for x in slots]
+        zz = torch.stack([torch.FloatTensor([b]).repeat(50,1).transpose(0,1)])
+        pred_slots = torch.mul(pred_slots, zz)
+
+    slt = [str(item) for batch_num, sublist in enumerate(pred_slots.max(1)[1].tolist()) for item in sublist[1:true_length[batch_num] + 1]]                                                                                                                      
+    out_slots = [slots[int(c)] for c in slt]
+
+    print("Input: {}\nIntent: {}\nSlots: {}".format(query, out_intent, out_slots))
+
+    print("--- %s seconds ---" % (time.time() - start_time))
+
+    # Write to output file
+    out = ""
+    collected_slots = {}
+    active_slot_words = []
+    active_slot_name = None
+    for words, slot_preds, intent_pred in zip([q.split()], [out_slots], [out_intent]):
+        line = ""
+        for word, pred in zip(words, slot_preds):
+            line = line + word + " "
+            if pred == 'O':
+                if active_slot_name:
+                    collected_slots[active_slot_name] = " ".join(active_slot_words)
+                    active_slot_words = []
+                    active_slot_name = None
             else:
-                collected_slots[active_slot_name] = " ".join(active_slot_words)
-                active_slot_words = [word]
-                active_slot_name = new_slot_name
-    out = line.strip()
-if active_slot_name:
-    collected_slots[active_slot_name] = " ".join(active_slot_words)
-
-print(collected_slots)
-print("--- %s seconds ---" % (time.time() - start_time))
+                # Naive BIO handling: treat B- and I- the same...
+                new_slot_name = pred[2:]
+                if active_slot_name is None:
+                    active_slot_words.append(word)
+                    active_slot_name = new_slot_name
+                elif new_slot_name == active_slot_name:
+                    active_slot_words.append(word)
+                else:
+                    collected_slots[active_slot_name] = " ".join(active_slot_words)
+                    active_slot_words = [word]
+                    active_slot_name = new_slot_name
+        out = line.strip()
+    if active_slot_name:
+        collected_slots[active_slot_name] = " ".join(active_slot_words)
+
+    print(collected_slots)
+    print("--- %s seconds ---" % (time.time() - start_time))
+
+predict("What's the weather like in York PA right now?")
+predict("How's the weather in York PA right now?")
+predict("What's the weather like in Great Mills right now?")
+predict("What will the weather be like in Frederick Maryland tomorrow?")
+predict("Play some jazz")
+predict("Play some daft punk")
+predict("Play some hatsune miku")