processors = {
      "cola": ColaProcessor,
      "mnli": MnliProcessor,
      "mrpc": MrpcProcessor,
      "xnli": XnliProcessor,
      "mytask": MyTaskProcessor,
  }

class MyTaskProcessor(DataProcessor):
    """Processor for the News data set (GLUE version)."""

    def __init__(self):
        self.labels = ['game', 'fashion', 'houseliving']
    
    def get_train_examples(self, data_dir):
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")
    
    def get_test_examples(self, data_dir):
    	return self._create_examples(
        	self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")

    def get_labels(self):
        return self.labels

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s" % (set_type, i)
            text_a = tokenization.convert_to_unicode(line[1])
            label = tokenization.convert_to_unicode(line[0])
            examples.append(
                InputExample(guid=guid, text_a=text_a, text_b=None, label=label))

        return examples

from bert_serving.client import BertClient
bc = BertClient()
bc.encode(['一二三四五六七八', '今天您吃了吗？'])

def mecab_to_text(sentence_list):
    """
    :param sentence_list: A list of sentences or one single sentence.
    :return: A list of segmented sentences.
    :note: Use mecab to segment a list of sentences or one single sentence in Japanese.
    """
    import MeCab
    mecab = MeCab.Tagger("-Ochasen")

    single_flag = False
    if isinstance(sentence_list, str):
        sentence_list = [sentence_list]
        single_flag = True

    ret_list = []
    for sentence in sentence_list:
        text_list = []
        m = mecab.parseToNode(sentence)
        while m:
            text_list.append(m.surface)
            m = m.next
        seg_sentence = " ".join(text_list).strip()
        ret_list.append(seg_sentence)

    if single_flag:
        return ret_list[0]
return ret_list

def load_text(use_length=-1, min_len=10):
    start = time.clock()
    japanese_text_path = "H:\\Work\\JapaneseModel\\Japanese_book\\"
    text_list = []
    
    if use_length == -1:
        for file in os.listdir(japanese_text_path):
            with open(japanese_text_path + file, 'r', encoding='utf-8') as f:
                for line in f.readlines():
                    line_use = line.strip()
                    if len(line_use) > min_len:
                        text_list.append(line_use)
    else:
        counter = 0
        for file in os.listdir(japanese_text_path):
            with open(japanese_text_path + file, 'r', encoding='utf-8') as f:
                for line in f.readlines():
                    line_use = line.strip()
                    if len(line_use) > min_len:
                        text_list.append(line_use)
                    counter += 1
                    if counter == use_length:
                        print("Japanese text loaded %d lines."%use_length)
                        elapsed = time.clock() - start
                        print("Time used:", round(elapsed, 3))
                        return text_list
                
    print("Japanese text loaded all lines.")
    elapsed = time.clock() - start
    print("Time used:", round(elapsed, 3))

    return text_list

def make_word_dictionary(split_text_list, lower_bound=100):
    start = time.clock()
    
    word_dictionary = dict()
    for sentence in split_text_list:
        sentence_use = sentence.split(" ")
        for word in sentence_use:
            if not word in word_dictionary:
                word_dictionary[word] = 1
            else:
                word_dictionary[word] += 1
                
    print("Word dictionary established.")
    elapsed = time.clock() - start
    print("Time used:", round(elapsed, 3))
    
    if lower_bound > 0:
        pop_list = []
        for word in word_dictionary:
            if word_dictionary[word] < lower_bound:
                pop_list.append(word)
        for word in pop_list:
            word_dictionary.pop(word)
            
    word_list = []
    for word in word_dictionary:
        word_list.append(word)
    
return word_list

def clear_dictionary(dictionary, embedding_dictionary):
    ret_list = []
    for word in dictionary:
        if word in embedding_dictionary:
            ret_list.append(word)
return ret_list

japanese_text = load_text(use_text_length)
split_japanese_text = mecab_to_text(japanese_text)
dictionary = make_word_dictionary(split_japanese_text, lower_bound=10)
dictionary = clear_dictionary(dictionary, embeddings_index)

def load_embedding():
    start = time.clock()
    
    """
    Total 2000000 words in this embedding file, 300-d. It is float16 type.
    The first line is "2000000 300".
    You should delete this line.
    """
    EMBEDDING_FILE = 'H:\\Work\\cc.ja.300.vec' 
    def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float16')
    embeddings_index = dict(get_coefs(*o.strip().split(" ")) for o in open(EMBEDDING_FILE, 'r', encoding="utf-8"))
    
    elapsed = time.clock() - start
    print("Word vectors loaded.")
    print("Time used:", round(elapsed, 3))
    
    return embeddings_index

def generate_train(window, end_index, text_seq):
    prefix = [0] * (window - 1)
    suffix = [end_index]
    x_list = []
    y_list = []
    for seq in text_seq:
        if len(seq) > 1:
            seq_use = prefix + seq + suffix
            # print(seq_use)
            for i in range(len(seq_use) - window + 1):
                x_list.append(seq_use[i: i + window - 1])
                y_list.append(seq_use[i + window - 1])
                # print(seq_use[i: i + window])
    return x_list, y_list

# Model
inp = Input(shape=(window - 1,))
x = Embedding(nb_words, 300, trainable = True, weights=[embedding_matrix])(inp)
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
x = Dropout(0.1)(x)
x = Bidirectional(CuDNNLSTM(128, return_sequences=False))(x)
x = Dropout(0.1)(x)
x = Dense(128, activation="relu")(x)
x = Dropout(0.1)(x)
x = Dense(nb_words, activation="softmax")(x)
model = Model(inputs=inp, outputs=x)
opt = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, 
                            epsilon=None, decay=0.0, amsgrad=False)
model.compile(loss='categorical_crossentropy', 
              optimizer=opt, metrics=['accuracy'])
history = LossHistory()

epoch_nb = 80 # 40 is enough
batch = 64

model.fit(x_train, y_train, batch_size=batch, epochs=epoch_nb, verbose=1,
          validation_data=(x_test, y_test), callbacks=[history])

def predict_random_sentence(new=[0] * (window - 1)):
    sentence = reverse_index[new[0]] + reverse_index[new[1]] + reverse_index[new[2]] + reverse_index[new[3]]
    while new[-1] != end_index:
        prob = model.predict(np.asarray([new]))[0]
        new_predict = int(random.choices(word_ind, weights=prob)[0])
        sentence += reverse_index[new_predict]
        new = new[1:] + [new_predict]
    return sentence

predict_random_sentence([0,0,0,0])

with open("../result/tokenizer.pkl", "wb") as handle:
    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
model.save('../model/language_model.model')

user_URL = 'https://sc.ftqq.com/' + SCKEY + '.send'

pip install pyserverchan

from pyserverchan import pyserver
svc = pyserver.ServerChan(user_URL)
svc.output_to_weixin("ATestMessage.")
svc.output_to_weixin_picture("http://sc.ftqq.com/static/image/bottom_logo.png")
svc.output_to_weixin_markdown("J:/pyserverchan/README.md")

import os
train_result = 'Accuracy on train:' + str(round(acc_train, 4) * 100) + "%" + os.linesep + os.linesep
train_result += 'Accuracy on test:' + str(round(acc_test, 4) * 100) + "%"
svc.output_to_weixin('Train done.', train_result) # 第一个参数是发送的文章标题，第二个参数是发送的文章内容