Prompt sounds for ESP32 AudioKit (ES8388 V1/V2) with Bluetooth A2DP source

[UnforeseenOcean]

Hello, I'm trying to make a Bluetooth speaker using ESP32-A2DP library (from the same author, latest) based on the example code given, and I want to use Audio Tools library to implement playing back prompt sounds based on the state A2DP reports. However I'm having trouble understanding the required code.

This code attached is currently non-functional and admittedly amateurish, but I hope it helps understanding what I'm trying to do.

// Placeholder functions are used. If something doesn't actually exist in the library, 
// think of it as "actual code goes here later" kind of statement. Any line marked with
// commented-out "P" means that line is intended to have them.

// #include <vorbis-tremor.h>
#include <CodecVorbis.h>
#include <OneButton.h>
#include "AudioTools.h"
#include "AudioTools/AudioLibs/AudioBoardStream.h" // install https://github.com/pschatzmann/arduino-audio-driver
#include "AudioTools/Disk/AudioSourceSPIFFS.h"
#include "BluetoothA2DPSink.h"
#include "FS.h" //P
#include "esp_spiffs.h" //P

#define KEY_VOLUP 18
#define KEY_NEXT 23
#define KEY_PLAY 19
#define KEY_PREV 13
#define KEY_VOLDN 36

// Filesystem definitions (Do not mix with below)
// Currently uses OGG but can be changed to WAV if resampling can be performed
#define FS_SND_VOL_MIN "vol-min.ogg"
#define FS_SND_VOL_MAX "vol-max.ogg"
#define FS_SND_BAT_LOW "battlow.ogg"
#define FS_SND_STA_CONNECT "connected.ogg"
#define FS_SND_STA_DISCONNECT "disconnected.ogg"
#define FS_SND_PWR_ON "poweron.ogg"
#define FS_SND_PWR_OFF "poweroff.ogg"
#define FS_SND_STA_LINK_LOSS "linkloss.ogg"
#define FS_SND_ERASE_PAIRED_DEVICES "init.ogg"
#define FS_SND_PAIR_ERROR "pair-fail.ogg"

// Index-only definitions (Do not mix with above)
#define SND_VOL_MIN 6
#define SND_VOL_MAX 7
#define SND_BAT_LOW 8
#define SND_STA_CONNECT 2
#define SND_STA_DISCONNECT 3
#define SND_PWR_ON 0
#define SND_PWR_OFF 1
#define SND_STA_LINK_LOSS 5
#define SND_ERASE_PAIRED_DEVICES 9
#define SND_PAIR_ERROR 4

int curVolume = 64;
bool isMuted = false;
uint16_t battTick = 0;
uint16_t battCheckTick = 3000;

AudioBoardStream kit(AudioKitEs8388V1); 
BluetoothA2DPSink a2dp_sink(kit);

AudioGeneratorOGG *snd; //P
AudioFileSourceSPIFFS *file; //P
AudioOutputI2S *out; //P

OneButton volup_btn(KEY_VOLUP, true);
OneButton next_btn(KEY_NEXT, true);
OneButton play_btn(KEY_PLAY, true);
OneButton prev_btn(KEY_PREV, true);
OneButton voldn_btn(KEY_VOLDN, true);

void setup() {
  //LOGLEVEL_AUDIOKIT = AudioKitInfo;
  Serial.begin(115200);

  if (!SPIFFS.begin(true)) {
    Serial.println("Error mounting SPIFFS volume");
    return;
  }

  // use medium volume
  kit.setVolume(1.0);
  // Enable below if the speaker's volume behavior is not acceptable to force it into a known state
  // a2dp_sink.set_volume(curVolume);

  // Setup buttons
  volup_btn.attachClick(volumeUpFine);
  volup_btn.attachDuringLongPress(volumeUpCoarse);
  next_btn.attachClick(goNext);
  play_btn.attachClick(goPlay);
  prev_btn.attachClick(goPrev);
  voldn_btn.attachClick(volumeDownFine);
  voldn_btn.attachDuringLongPress(volumeDownCoarse);

  // Set delays
  volup_btn.setDebounceMs(100);
  next_btn.setDebounceMs(100);
  play_btn.setDebounceMs(100);
  prev_btn.setDebounceMs(100);
  voldn_btn.setDebounceMs(100);

  volup_btn.setClickMs(500);
  next_btn.setClickMs(500);
  play_btn.setClickMs(500);
  prev_btn.setClickMs(500);
  voldn_btn.setClickMs(500);

  volup_btn.setPressMs(1000);
  next_btn.setPressMs(1000);
  play_btn.setPressMs(1000);
  prev_btn.setPressMs(1000);
  voldn_btn.setPressMs(1000);

  // start a2dp
  a2dp_sink.start("ESP-BT-SPK");  
  playSound(FS_SND_PWR_ON);

}

void volumeUpFine() {
  curVolume = a2dp_sink.get_volume();
  curVolume = curVolume + 5;
  if (curVolume > 127) { 
    curVolume = 127;
    playSound(FS_SND_VOL_MAX);
  }
  a2dp_sink.set_volume(curVolume);
}

void volumeUpCoarse() {
  curVolume = a2dp_sink.get_volume();
  curVolume = curVolume + 10;
  if (curVolume > 127) { 
    curVolume = 127;
    playSound(FS_SND_VOL_MAX);
  }
  a2dp_sink.set_volume(curVolume);
}

void volumeDownFine() {
  curVolume = a2dp_sink.get_volume();
  curVolume = curVolume - 5;
  if (curVolume < 0) { 
    curVolume = 0;
    playSound(FS_SND_VOL_MIN);
  }
  a2dp_sink.set_volume(curVolume);
}

void volumeDownCoarse() {
  curVolume = a2dp_sink.get_volume();
  curVolume = curVolume - 10;
  if (curVolume < 0) { 
    curVolume = 0;
    playSound(FS_SND_VOL_MIN);
  }
  a2dp_sink.set_volume(curVolume);
}



void playSound(const char* filename) {
  // TODO: Implement SPIFFS/C Header WAV playback
  // Need to switch between WAV source and A2DP audio source with upsampling if needed
  // Or can use OGG Vorbis... I guess?
  //    [Prompt Sound Output]-----(I2S Stream 2)----.
  //                                                |
  // [Bluetooth AVRCP]---->(I2S Stream 1)---->[I2S Switcher]---->[I2S Output]---->[I2S DAC]
  
  // Stop playing previous sound if a sound is already playing
  if (snd && snd->isRunning()) { //P
    snd->stop(); //P
    delete snd; //P
    delete file; //P
    delete out; //P
  }
  file = new AudioFileSourceSPIFFS(filename); //P
  out = new AudioOutputI2S(); //P
  out->SetGain(0.5); //P, Prompt tone volume
  snd = new AudioGeneratorOGG(); //P
  snd->begin(file, out); //P
}

void goNext() {
  a2dp_sink.next();
}

void goPlay() {
  a2dp_sink.play();
}

void goPrev() {
  a2dp_sink.previous();
}

void mute() {
  if (isMuted) {
    a2dp_sink.set_volume(curVolume);
  }
  else {
    a2dp_sink.set_volume(0);
  }
  isMuted = !isMuted;
}

int checkBatteryLevel() {
  float threshold = 3.53;
  // ADC reading code goes here...

  if (result < threshold) {
    playSound(FS_SND_BAT_LOW);
  }
}

void loop() {
  delay(10);
  volup_btn.tick();
  next_btn.tick();
  play_btn.tick();
  prev_btn.tick();
  voldn_btn.tick();
  battTick++;
  if (battTick >= battCheckTick) {
    checkBatteryLevel();
    battTick = 0;
  }
}

[pschatzmann]

Please read the chapter about Codecs: instead of doing coper.copy() in the loop you can just call copier.copyAll() to output the whole file.
I also suggest to store the data in PROGMEM and using a MemoryStream as described in this Wiki instead of using SPIFFS.

Just make sure that your playback is not changing the sample rate or channels: otherwise you need to restore it to the original values!

[UnforeseenOcean]

Alright, things are not working out right now. Let me clarify some points and issues I have:

• I need an example code of A2DP stream being used with another stream.
• I don't mind how it's done, but the sound that will play replaces the stream going to the sink until it's done playing. Imagine a PA speaker that is playing background music, that's one of the similar cases.
• Concepts and low-level recommendations will not help matters for now. I need an example code of some kind.

Could you help me, please?

[pschatzmann]

Your question that you are asking for an example of an "A2DP stream being used with another stream" does not make any sense.
Since the beginning, I am telling you that the source is irrelevant.
You can easily build an example with 2 sources yourself.

1. take the A2DPStream example that I gave you as starting point.
2. add another data source (e.g. GeneratedSoundStream with a SineWaveGenerator)
3. add another StreamCopy from the GeneratedSoundStream to the AudioBoardStream
4. in the loop decide which copier to use

[UnforeseenOcean]

I think the problem stems from the fact that I don't know how this whole copying business works in the slightest. Does the StreamCopy object take in more than two arguments -- input and output? Is it a shared pool of sorts instead which takes in input from various sources and let each source take turns? What is it? "a typeless StreamCopy" does not explain anything.
I know you're probably as equally, or even more than I am, frustrated with me, but please throw me a bone, I am trying hard to do things on my own and yet, I feel utterly defeated because I don't know what anything in the library does anymore. I plead to you, please show me the way, sir.

[pschatzmann]

Just read the Introduction Wiki, if you don't understand the basic concepts!
There is also a class documentation for each class that show all methods and gives a short explanation...

[UnforeseenOcean]

OK, tell me one thing: If I try to send data to a sink (or a copy object that's sending data to the sink, whatever floats the boat) that is already receiving data, what happens next?

a) The sound stops playing altogether
b) The sound is overwritten by second data being sent to the sink, and does not return when it ends
c) The sound is temporarily replaced by the second data stream, and immediately returns to the first data stream when it ends
d) This is not allowed, the processor crashes/the sound gets corrupted/etc.
e) Something else entirely

The prerequisite is that both streams are identical in their format and sampling rate (44100Hz 16bit PCM with two channels, Left and Right)
I am trying to confirm something, please cooperate please. Only a simple explanation is required this time.

[pschatzmann]

You can consider I2S as a blocking queue: If you write data to it you just add more data to the end of the queue. Each copy step has 1kbytes (by default) which is only 256 stereo audio samples. At 44100 samples per second this is only playing for 5.8 milliseconds!

If you don't provide the data fast enough, the sound is breaking up at stuttering...
If you don't provide any data, it just plays wha't in the (small) queue and then stops playing

[UnforeseenOcean]

So, does this mean that I can just send the data to that object and it will replace the sound while it is playing? Or did I misunderstand it again?

[pschatzmann]

Theoretically it is not replacing, but practically you will not notice the few ms of old data playing

[UnforeseenOcean]

(Final confirmation) So the case A happens and not case B shown here, right?

[joba-1]

That depends on your input stream:

• If your input is e.g. a i2s mic or a2dp input, then samples get discarded while they are not processed during prompt play and you get A.
• If your input is a file or a memory stream it will continue reading the file or memory where it was when the prompt started, so you get B. To avoid that you would have to advance the read position of the file/memory (seeking forward as many samples as the prompt had)

[UnforeseenOcean]

Now the problem is an approach to playing multiple sounds, one at a time, on demand. These are my best interpretations of the library, does any of it sound plausible?

1. Make one decoder object (in these examples it will be called "ogg"), switch out the input file (which is defined in the headers), then invoke play();
2. Make multiple decoder objects (ogg0~ogg9 for example) and point them at the sink, then invoke play(); on decoders with sound I want

Plus, does A2DPStream support getting the status (connection status to be more exact) from the connection, or is it supposed to be used with BluetoothA2DPSink.h?

[pschatzmann]

Before deciding on a decoder and approach, check how much RAM you have left when A2DP playback is active and measure how much RAM your decoder is using at playback!

[UnforeseenOcean]

Sir, I know you're trying to help, but without any implementation plans, it would be near impossible to analyze the required parameters. Please define one implementation for me to attempt.

[UnforeseenOcean]

I think I am starting to slowly understand what you meant, now that I combed through hundreds and thousands of lines of code. I sincerely apologize for any inconvenience. This is what I have currently understood:

• I can send two or more data streams to one sink, which will have the output replaced with the latest data stream
• VorbisDecoder does not need StreamCopy(output, input) but others may need it
• No logic is required for the prompt sound (that is, no switching is needed)

Now all that remains is to get an answer to this question:
How do you play different audio samples from PROGMEM on demand?

Currently, OneButton, AudioTools, AudioBoardStream, CodecVorbis, BluetoothA2DPSink, and the sounds are being used, with BluetoothA2DPSink being used because I can't find a way to get the callback events and A2DP commands from A2DPStream. (If I find a way to use callback events on A2DPStream, I will change over to that part of the library)

[UnforeseenOcean]

It is quite clear to me that I won't get help through this way. Since there is not an example code for relevant part of the library, I am unable to continue. Have a nice day.