diff --git a/.gitignore b/.gitignore index 2591f2d..830f5a2 100644 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,6 @@ glove/ venv/ env/ glove.6B.zip +.vscode +.DS_Store +*.pt \ No newline at end of file diff --git a/datasets/#count.py# b/datasets/#count.py# deleted file mode 100644 index 023ecbe..0000000 --- a/datasets/#count.py# +++ /dev/null @@ -1,10 +0,0 @@ -filename = 'atis/train/seq.out' - -blah = set() -with open(filename) as f: - for line in f: - line = line.rstrip().split() - for word in line: - blah.add(word) - -print(len(blah)) diff --git a/datasets/snips/train/label b/datasets/snips/train/label index 7545e21..5e3004c 100644 --- a/datasets/snips/train/label +++ b/datasets/snips/train/label @@ -13082,3 +13082,5 @@ PlayMusic RateBook SearchCreativeWork RateBook +GetWeather +GetWeather \ No newline at end of file diff --git a/datasets/snips/train/seq.in b/datasets/snips/train/seq.in index c3abd55..dbd639e 100644 --- a/datasets/snips/train/seq.in +++ b/datasets/snips/train/seq.in @@ -13082,3 +13082,5 @@ play funky heavy bluesy rate the current album 2 points out of 6 go to the photograph the inflated tear rate richard carvel 4 out of 6 +how does the weather look in york pa right now +what s the weather like in new market \ No newline at end of file diff --git a/datasets/snips/train/seq.out b/datasets/snips/train/seq.out index b14967e..578a4a5 100644 --- a/datasets/snips/train/seq.out +++ b/datasets/snips/train/seq.out @@ -13082,3 +13082,5 @@ O B-playlist I-playlist I-playlist O O B-object_select B-object_type B-rating_value B-rating_unit O O B-best_rating O O O B-object_type B-object_name I-object_name I-object_name O B-object_name I-object_name B-rating_value O O B-best_rating +O O O O O O B-city B-state O B-timeRange +O O O O O O B-city I-city \ No newline at end of file diff --git a/dump_pickles.py b/dump_pickles.py new file mode 100644 index 0000000..388275f --- /dev/null +++ b/dump_pickles.py @@ -0,0 +1,15 @@ +from dataset import Corpus, build_glove +import pickle + +train_corpus = Corpus('snips', 'train', 50) + +pickle.dump(train_corpus.word2idx, open( "word2idx.pkl", "wb" )) + +wordvecs = build_glove(train_corpus.word2idx, train_corpus.idx2word) +pickle.dump(wordvecs, open( "wordvecs.pkl", "wb" )) + +slots = [slot for slot in train_corpus.slot2idx] +pickle.dump(slots, open( "slots.pkl", "wb" )) + +intent = [intent for intent in train_corpus.intent2idx] +pickle.dump(intent, open( "intents.pkl", "wb" )) \ No newline at end of file diff --git a/intent_filters.py b/intent_filters.py new file mode 100644 index 0000000..3a41a80 --- /dev/null +++ b/intent_filters.py @@ -0,0 +1,46 @@ +import pickle +# zz = torch.stack([z.repeat(50,1).transpose(0,1)]) + +# GetWeather +getWeather = ['state', 'city', 'timeRange', 'geographic_poi', 'country', 'facility'] + +# PlayMusic +playMusic = ['artist', 'album', 'playlist', 'service', 'rating_value', 'best_rating', 'music_item', 'track', 'playlist_owner', 'year', 'genre', 'rating_unit'] + +# AddToPlaylist +addToPlaylist = ['artist', 'album', 'playlist', 'service', 'rating_value', 'best_rating', 'music_item', 'track', 'playlist_owner', 'year', 'genre', 'rating_unit'] + +# RateBook +## I don't care + +# SearchScreeningEvent +## I don't care + +# BookRestaurant +## I don't care + +# SearchCreativeWork +## I don't care + +generic = ['O', 'B-entity_name', 'B-object_name', 'B-object_type', 'I-entity_name', 'I-object_name', 'I-object_type'] + +getWeatherOut = generic[:] +for i in getWeather: + getWeatherOut.append('B-{}'.format(i)) + getWeatherOut.append('I-{}'.format(i)) + +playMusicOut = generic[:] +for i in playMusic: + playMusicOut.append('B-{}'.format(i)) + playMusicOut.append('I-{}'.format(i)) + +addToPlaylistOut = generic[:] +for i in addToPlaylist: + addToPlaylistOut.append('B-{}'.format(i)) + addToPlaylistOut.append('I-{}'.format(i)) + +out = {"GetWeather": getWeatherOut, + "PlayMusic": playMusicOut, + "AddToPlaylist": addToPlaylist} + +pickle.dump(out, open("slot_filters.pkl", "wb")) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e4f0076..0d72d24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -torch +torch==1.5.1 seqeval scikit-learn transformers diff --git a/slot_filters.pkl b/slot_filters.pkl new file mode 100644 index 0000000..56ee073 Binary files /dev/null and b/slot_filters.pkl differ diff --git a/test_query.py b/test_query.py index e2febb7..98709be 100644 --- a/test_query.py +++ b/test_query.py @@ -17,10 +17,11 @@ EOS = "" word2idx = pickle.load(open("word2idx.pkl", "rb")) wordvecs = pickle.load(open("wordvecs.pkl", "rb")) slots = pickle.load(open("slots.pkl", "rb")) +slot_filters = pickle.load(open("slot_filters.pkl", "rb")) intents = pickle.load(open("intents.pkl", "rb")) num_words = len(word2idx) -num_intent = 7 -num_slot = 72 +num_intent = len(intents) +num_slot = len(slots) filter_count = 300 dropout = 0 embedding_dim = 100 @@ -33,63 +34,71 @@ def pad_query(sequence): return sequence -query = "What's the weather like in York PA right now?" -q = query.lower().replace("'", " ").replace("?", " ").strip() -# true_length = [len(q.split()), 0, 0, 0, 0, 0, 0 ,0] -true_length = [len(q.split())] -qq = torch.from_numpy(pad_query([word2idx[word] for word in q.split()])) - -model = models.CNNJoint(num_words, embedding_dim, num_intent, num_slot, (filter_count,), 5, dropout, wordvecs) -model.eval() - -model.load_state_dict(torch.load('snips_joint', map_location=torch.device('cpu'))) -criterion = torch.nn.CrossEntropyLoss(ignore_index=-1) - -pad_tensor = torch.from_numpy(pad_query([word2idx[w] for w in []])) -# batch = torch.stack([qq, pad_tensor, pad_tensor, pad_tensor, pad_tensor, pad_tensor, pad_tensor, pad_tensor]) -batch = torch.stack([qq]) - -pred_intent, pred_slots = model(batch) - -slt = [str(item) for batch_num, sublist in enumerate(pred_slots.max(1)[1].tolist()) for item in sublist[1:true_length[batch_num] + 1]] -out_slots = [slots[int(c)] for c in slt] - -itnt = pred_intent.max(1)[1].tolist()[0] -out_intent = intents[itnt] - -print("Input: {}\nIntent: {}\nSlots: {}".format(query, out_intent, out_slots)) - -print("--- %s seconds ---" % (time.time() - start_time)) - -# Write to output file -out = "" -collected_slots = {} -active_slot_words = [] -active_slot_name = None -for words, slot_preds, intent_pred in zip([q.split()], [out_slots], [out_intent]): - line = "" - for word, pred in zip(words, slot_preds): - line = line + word + " " - if pred == 'O': - if active_slot_name: - collected_slots[active_slot_name] = " ".join(active_slot_words) - active_slot_words = [] - active_slot_name = None - else: - # Naive BIO handling: treat B- and I- the same... - new_slot_name = pred[2:] - if active_slot_name is None: - active_slot_words.append(word) - active_slot_name = new_slot_name - elif new_slot_name == active_slot_name: - active_slot_words.append(word) +def predict(query): + q = query.lower().replace("'", " ").replace("?", " ").strip() + true_length = [len(q.split())] + qq = torch.from_numpy(pad_query([word2idx[word] if word in word2idx else word2idx[""] for word in q.split()])) + + model = models.CNNJoint(num_words, embedding_dim, num_intent, num_slot, (filter_count,), 5, dropout, wordvecs) + model.eval() + + model.load_state_dict(torch.load('snips_joint', map_location=torch.device('cpu'))) + + batch = torch.stack([qq]) + pred_intent, pred_slots = model(batch) + + itnt = pred_intent.max(1)[1].tolist()[0] + out_intent = intents[itnt] + + if out_intent in slot_filters: + b = [1 if x in slot_filters[out_intent] else 0 for x in slots] + zz = torch.stack([torch.FloatTensor([b]).repeat(50,1).transpose(0,1)]) + pred_slots = torch.mul(pred_slots, zz) + + slt = [str(item) for batch_num, sublist in enumerate(pred_slots.max(1)[1].tolist()) for item in sublist[1:true_length[batch_num] + 1]] + out_slots = [slots[int(c)] for c in slt] + + print("Input: {}\nIntent: {}\nSlots: {}".format(query, out_intent, out_slots)) + + print("--- %s seconds ---" % (time.time() - start_time)) + + # Write to output file + out = "" + collected_slots = {} + active_slot_words = [] + active_slot_name = None + for words, slot_preds, intent_pred in zip([q.split()], [out_slots], [out_intent]): + line = "" + for word, pred in zip(words, slot_preds): + line = line + word + " " + if pred == 'O': + if active_slot_name: + collected_slots[active_slot_name] = " ".join(active_slot_words) + active_slot_words = [] + active_slot_name = None else: - collected_slots[active_slot_name] = " ".join(active_slot_words) - active_slot_words = [word] - active_slot_name = new_slot_name - out = line.strip() -if active_slot_name: - collected_slots[active_slot_name] = " ".join(active_slot_words) - -print(collected_slots) -print("--- %s seconds ---" % (time.time() - start_time)) \ No newline at end of file + # Naive BIO handling: treat B- and I- the same... + new_slot_name = pred[2:] + if active_slot_name is None: + active_slot_words.append(word) + active_slot_name = new_slot_name + elif new_slot_name == active_slot_name: + active_slot_words.append(word) + else: + collected_slots[active_slot_name] = " ".join(active_slot_words) + active_slot_words = [word] + active_slot_name = new_slot_name + out = line.strip() + if active_slot_name: + collected_slots[active_slot_name] = " ".join(active_slot_words) + + print(collected_slots) + print("--- %s seconds ---" % (time.time() - start_time)) + +predict("What's the weather like in York PA right now?") +predict("How's the weather in York PA right now?") +predict("What's the weather like in Great Mills right now?") +predict("What will the weather be like in Frederick Maryland tomorrow?") +predict("Play some jazz") +predict("Play some daft punk") +predict("Play some hatsune miku") \ No newline at end of file