When I use multi GPUs to train, but I want to conduct validation test during the train, how can I realize it? Here is my code:

if ckpt is None:
    logging.info("Starting new training")
    sess.run(tf.global_variables_initializer())
    sess.run(bcast)
else:
    logging.info("Resuming from checkpoint: %s" % ckpt)
    saver.restore(sess, ckpt)

while True:
    try:
        ids, _gs, _loss, _acc, _summary, _ = sess.run([train_id, global_step, loss, accuracy, train_summary, train_op])
        if hvd.rank() == 0:
            logging.info("step {}, loss:{:.4f}, accuracy:{:.4f}".format(_gs, _loss, _acc))

            if _gs % 10 == 0:
                summary_writer.add_summary(_summary, _gs)

            if _gs % 1000 == 0:
                logging.info("# save models at {} step".format(_gs))
                saver.save(sess, ckpt_name, global_step=_gs)

            if math.isnan(_loss):
                logging.info("第hvd.rank({})个进程的第{}步出现错误".fromat(ids, _gs))
                raise Exception('Loss Exploded')

            if _gs % hp.eval_per_step == 0:
                logging.info("# statrt a validation test: ")
                ......