MMoE

expert_output_tensor

concat

gate

subtask_input=tf.squeeze(gate_softmax*expert_output_tensor)

task

expert2

combined_score_sigmoid = sigmoid(output_score)

gate_softmax = expand_dims(gate_softmax)

ExpertLayer

expert1

subtask_input

loss: ctr: cross_entropydwell: square_error   error = output_score - sample_weightlosses =  loss_ops[ctr] + loss_ops[dwell]loss_ops = (l2_losses + task_layer_l2_loss) * l2_loss_coefl2_losses = sum(l2_list)task_layer_l2_loss = l2_loss(w_gate) + l2_loss(w_top)

expert_output

dense_layer

output_score = subtask_input*w_top +b_top

gate_softmax = softmax(expert_output *w_gate))

expert3