Chat with batch

[MartinPerry]

Can I use llama_chat_apply_template in batched processing? I mean, to pass multiple prompts (each of them different) and receive multiple results (one for each prompt). Currently, based on batch example, I can get multiple responses for a single prompt (however, they have slightly messed up order or words in comparison to only one batch).
My current main code to process data from llama_chat_apply_template looks like this:

void Loop(std::string prompt) {
  std::vector<llama_token> prompt_tokens = Tokenize(prompt);
  
  size_t n_parallel = 4;
  llama_batch batch = llama_batch_init(std::max(prompt_tokens.size(), n_parallel), 0, n_parallel);
  
  {
      std::vector<llama_seq_id> seq_ids(n_parallel, 0);
      for (int32_t i = 0; i < n_parallel; ++i)
      {
          seq_ids[i] = i;
      }
  
      for (size_t i = 0; i < prompt_tokens.size(); ++i)
      {
          batch.token[batch.n_tokens] = prompt_tokens[i];
          batch.pos[batch.n_tokens] = i;
          batch.n_seq_id[batch.n_tokens] = seq_ids.size();
          for (size_t i = 0; i < seq_ids.size(); ++i)
          {
              batch.seq_id[batch.n_tokens][i] = seq_ids[i];
          }
          batch.logits[batch.n_tokens] = false;
  
          batch.n_tokens++;
  
          //common_batch_add(batch, tokens_list[i], i, seq_ids, false);
      }
  }
  
  batch.logits[batch.n_tokens - 1] = true;
  
  std::vector<std::string> streams(n_parallel);
  std::vector<int32_t> i_batch(n_parallel, batch.n_tokens - 1);
  
  int n_cur = batch.n_tokens;
  bool isFinished = false;
  
  while (isFinished == false)
  {        
      if (llama_decode(ctx, batch))
      {
          GGML_ABORT("failed to decode\n");
      }
      batch.n_tokens = 0;
      for (int32_t i = 0; i < n_parallel; ++i)
      {
          if (i_batch[i] < 0)
          {
              continue;
          }
  
          llama_token new_token_id = llama_sampler_sample(sampler, ctx, i_batch[i]);
  
          if (llama_vocab_is_eog(vocab, new_token_id))
          {
              isFinished = true;
              break;
          }
       
          std::string tmp = TokenToString(new_token_id);
          streams[i] += tmp;
  
          i_batch[i] = batch.n_tokens;
  
          batch.token[batch.n_tokens] = new_token_id;
          batch.pos[batch.n_tokens] = n_cur;
          batch.n_seq_id[batch.n_tokens] = 1;
          batch.seq_id[batch.n_tokens][0] = i;
          batch.logits[batch.n_tokens] = true;
  
          batch.n_tokens++;
      }
  }
  
  std::string response = "";
  for (int32_t i = 0; i < n_parallel; ++i)
  {
      response += streams[I];
      response += "\n";
  }
}