/******************************************************************************* * Backend.cpp * *-------------* * Description: * This module is the implementation file for the CBackend class. *------------------------------------------------------------------------------- * Created By: mc Date: 03/12/99 * Copyright (C) 1999 Microsoft Corporation * All Rights Reserved * *******************************************************************************/ #include "stdafx.h" #ifndef __spttseng_h__ #include "spttseng.h" #endif #ifndef Backend_H #include "Backend.h" #endif #ifndef FeedChain_H #include "FeedChain.h" #endif #ifndef SPDebug_h #include #endif //----------------------------- // Data.cpp //----------------------------- extern const short g_IPAToAllo[]; extern const short g_AlloToViseme[]; //-------------------------------------- // DEBUG: Save utterance WAV file //-------------------------------------- //#define SAVE_WAVE_FILE 1 const unsigned char g_SineWaveTbl[] = { 0x7b,0x7e,0x81,0x84,0x87,0x89,0x8c,0x8f,0x92,0x95,0x98,0x9b,0x9d,0xa0,0xa3,0xa6, 0xa8,0xab,0xae,0xb0,0xb3,0xb5,0xb8,0xbb,0xbd,0xbf,0xc2,0xc4,0xc7,0xc9,0xcb,0xcd, 0xcf,0xd1,0xd3,0xd5,0xd7,0xd9,0xdb,0xdd,0xdf,0xe0,0xe2,0xe3,0xe5,0xe6,0xe8,0xe9, 0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf2,0xf3,0xf3,0xf4,0xf4,0xf4,0xf4, 0xf5,0xf5,0xf5,0xf5,0xf4,0xf4,0xf4,0xf4,0xf3,0xf3,0xf2,0xf1,0xf1,0xf0,0xef,0xee, 0xed,0xec,0xeb,0xea,0xe9,0xe7,0xe6,0xe5,0xe3,0xe1,0xe0,0xde,0xdc,0xdb,0xd9,0xd7, 0xd5,0xd3,0xd1,0xcf,0xcd,0xcb,0xc8,0xc6,0xc4,0xc1,0xbf,0xbc,0xba,0xb7,0xb5,0xb2, 0xb0,0xad,0xaa,0xa8,0xa5,0xa2,0x9f,0x9d,0x9a,0x97,0x94,0x91,0x8f,0x8c,0x89,0x86, 0x83,0x80,0x7d,0x7a,0x77,0x75,0x72,0x6f,0x6c,0x69,0x66,0x64,0x61,0x5e,0x5b,0x58, 0x56,0x53,0x50,0x4e,0x4b,0x49,0x46,0x44,0x41,0x3f,0x3c,0x3a,0x38,0x35,0x33,0x31, 0x2f,0x2d,0x2b,0x29,0x27,0x25,0x23,0x21,0x1f,0x1e,0x1c,0x1b,0x19,0x18,0x16,0x15, 0x14,0x13,0x12,0x11,0x10,0x0f,0x0e,0x0d,0x0c,0x0c,0x0b,0x0b,0x0a,0x0a,0x0a,0x0a, 0x09,0x09,0x09,0x09,0x0a,0x0a,0x0a,0x0a,0x0b,0x0b,0x0c,0x0d,0x0d,0x0e,0x0f,0x10, 0x11,0x12,0x13,0x14,0x15,0x17,0x18,0x1a,0x1b,0x1d,0x1e,0x20,0x22,0x23,0x25,0x27, 0x29,0x2b,0x2d,0x2f,0x31,0x34,0x36,0x38,0x3a,0x3d,0x3f,0x42,0x44,0x47,0x49,0x4c, 0x4e,0x51,0x54,0x56,0x59,0x5c,0x5f,0x61,0x64,0x67,0x6a,0x6d,0x6f,0x72,0x75,0x78 }; /*void PredictEpochDist( float duration, long nKnots, float SampleRate, float *pTime, float *pF0) { long curSamplesOut, endSample, j; float epochFreq; long epochLen, epochCount; curSamplesOut = 0; endSample = (long) (SampleRate * duration ); epochCount = 0; while( curSamplesOut < endSample ) { j = 1; //--------------------------------------------------- // Align to appropriate knot bassed on // current output sample //--------------------------------------------------- while( (j < nKnots - 1) && (curSamplesOut > pTime[j]) ) j++; //--------------------------------------------------- // Calculate exact pitch thru linear interpolation //--------------------------------------------------- epochFreq = LinInterp( pTime[j - 1], curSamplesOut, pTime[j], pF0[j - 1], pF0[j] ); //--------------------------------------------------- // Calc sample count for curent epoch //--------------------------------------------------- epochLen = (long) (SampleRate / epochFreq); epochCount++; curSamplesOut += epochLen; } } */ /***************************************************************************** * CBackend::CBackend * *--------------------* * Description: Constructor * ********************************************************************** MC ***/ CBackend::CBackend( ) { SPDBG_FUNC( "CBackend::CBackend" ); m_pHistory = NULL; m_pHistory2 = NULL; m_pFilter = NULL; m_pReverb = NULL; m_pOutEpoch = NULL; m_pMap = NULL; m_pRevFlag = NULL; m_pSpeechBuf = NULL; m_VibratoDepth = 0; m_UnitVolume = 1.0f; m_MasterVolume = SPMAX_VOLUME; memset( &m_Synth, 0, sizeof(MSUNITDATA) ); } /* CBackend::CBackend */ /***************************************************************************** * CBackend::~CBackend * *---------------------* * Description: Destructor * ********************************************************************** MC ***/ CBackend::~CBackend( ) { SPDBG_FUNC( "CBackend::~CBackend" ); Release(); } /* CBackend::~CBackend */ /***************************************************************************** * CBackend::Release * *---------------------* * Description: * Free memory allocaterd by Backend * ********************************************************************** MC ***/ void CBackend::Release( ) { SPDBG_FUNC( "CBackend::Release" ); CleanUpSynth( ); if( m_pSpeechBuf) { delete m_pSpeechBuf; m_pSpeechBuf = NULL; } if( m_pHistory ) { delete m_pHistory; m_pHistory = NULL; } if( m_pHistory2 ) { delete m_pHistory2; m_pHistory2 = NULL; } if( m_pReverb ) { delete m_pReverb; m_pReverb = NULL; } } /* CBackend::Release */ /***************************************************************************** * CBackend::Init * *----------------* * Description: * Opens a backend instance, keeping a pointer of the acoustic * inventory. * ********************************************************************** MC ***/ HRESULT CBackend::Init( IMSVoiceData* pVoiceDataObj, CFeedChain *pSrcObj, MSVOICEINFO* pVoiceInfo ) { SPDBG_FUNC( "CBackend::Init" ); long LPCsize = 0; HRESULT hr = S_OK; m_pVoiceDataObj = pVoiceDataObj; m_SampleRate = (float)pVoiceInfo->SampleRate; m_pSrcObj = pSrcObj; m_cOrder = pVoiceInfo->LPCOrder; m_pWindow = pVoiceInfo->pWindow; m_FFTSize = pVoiceInfo->FFTSize; m_VibratoDepth = ((float)pVoiceInfo->VibratoDepth) / 100.0f; m_VibratoDepth = 0; // NOTE: disable vibrato m_VibratoFreq = pVoiceInfo->VibratoFreq; if( pVoiceInfo->eReverbType > REVERB_TYPE_OFF ) { m_StereoOut = true; m_BytesPerSample = 4; } else { m_StereoOut = false; m_BytesPerSample = 2; } //--------------------------------------- // Allocate AUDIO buffer //--------------------------------------- m_pSpeechBuf = new float[SPEECH_FRAME_SIZE + SPEECH_FRAME_OVER]; if( m_pSpeechBuf == NULL ) { //-------------------------------------- // Out of memory! //-------------------------------------- hr = E_OUTOFMEMORY; } if( SUCCEEDED(hr) ) { //--------------------------------------- // Allocate HISTORY buffer //--------------------------------------- LPCsize = m_cOrder + 1; m_pHistory = new float[LPCsize]; if( m_pHistory == NULL ) { //-------------------------------------- // Out of memory! //-------------------------------------- hr = E_OUTOFMEMORY; } } if( SUCCEEDED(hr) ) { memset( m_pHistory, 0, LPCsize * sizeof(float) ); m_pOutEpoch = NULL; m_pMap = NULL; m_pRevFlag = NULL; m_fModifiers = 0; m_vibrato_Phase1 = 0; //-------------------------------- // Reverb Effect //-------------------------------- //pVoiceInfo->eReverbType = REVERB_TYPE_HALL; if( pVoiceInfo->eReverbType > REVERB_TYPE_OFF ) { //-------------------------------- // Create ReverbFX object //-------------------------------- if( m_pReverb == NULL ) { m_pReverb = new CReverbFX; if( m_pReverb ) { short result; result = m_pReverb->Reverb_Init( pVoiceInfo->eReverbType, (long)m_SampleRate, m_StereoOut ); if( result != KREVERB_NOERROR ) { //-------------------------------------------- // Not enough memory to do reverb // Recover gracefully //-------------------------------------------- delete m_pReverb; m_pReverb = NULL; } /*else { //-------------------------------------------------------- // Init was successful, ready to do reverb now //-------------------------------------------------------- }*/ } } } //---------------------------- // Linear taper region scale //---------------------------- m_linearScale = (float) pow( 10.0, (double)((1.0f - LINEAR_BKPT) * LOG_RANGE) / 20.0 ); #ifdef SAVE_WAVE_FILE m_SaveFile = (PCSaveWAV) new CSaveWAV; // No check needed, if this fails, we simply don't save file. if( m_SaveFile ) { m_SaveFile->OpenWavFile( (long)m_SampleRate ); } #endif } else { if( m_pSpeechBuf ) { delete m_pSpeechBuf; m_pSpeechBuf = NULL; } if( m_pHistory ) { delete m_pHistory; m_pHistory = NULL; } } return hr; } /* CBackend::Init */ /***************************************************************************** * CBackend::FreeSynth * *---------------------* * Description: * Return TRUE if consoants can be clustered. * ********************************************************************** MC ***/ void CBackend::FreeSynth( MSUNITDATA* pSynth ) { SPDBG_FUNC( "CBackend::FreeSynth" ); if( pSynth->pEpoch ) { delete pSynth->pEpoch; pSynth->pEpoch = NULL; } if( pSynth->pRes ) { delete pSynth->pRes; pSynth->pRes = NULL; } if( pSynth->pLPC ) { delete pSynth->pLPC; pSynth->pLPC = NULL; } } /* CBackend::FreeSynth */ /***************************************************************************** * ExpConverter * *--------------* * Description: * Convert linear to exponential taper * 'ref' is a linear value between 0.0 to 1.0 * ********************************************************************** MC ***/ static float ExpConverter( float ref, float linearScale ) { SPDBG_FUNC( "ExpConverter" ); float audioGain; if( ref < LINEAR_BKPT) { //---------------------------------------- // Linear taper below LINEAR_BKPT //---------------------------------------- audioGain = linearScale * (ref / LINEAR_BKPT); } else { //---------------------------------------- // Log taper above LINEAR_BKPT //---------------------------------------- audioGain = (float) pow( 10.0, (double)((1.0f - ref) * LOG_RANGE) / 20.0 ); } return audioGain; } /* ExpConverter */ /***************************************************************************** * CBackend::CvtToShort * *----------------------* * Description: * Convert (in place) FLOAT audio to SHORT. * ********************************************************************** MC ***/ void CBackend::CvtToShort( float *pSrc, long blocksize, long stereoOut, float audioGain ) { SPDBG_FUNC( "CBackend::CvtToShort" ); long i; short *pDest; float fSamp; pDest = (short*)pSrc; for( i = 0; i < blocksize; ++i ) { //------------------------ // Read float sample... //------------------------ fSamp = (*pSrc++) * audioGain; //------------------------ // ...clip to 16-bits... //------------------------ if( fSamp > 32767 ) { fSamp = 32767; } else if( fSamp < (-32768) ) { fSamp = (-32768); } //------------------------ // ...save as SHORT //------------------------ *pDest++ = (short)fSamp; if( stereoOut ) { *pDest++ = (short)(0 - (int)fSamp); } } } /* CBackend::CvtToShort */ /***************************************************************************** * CBackend::PSOLA_Stretch * *-------------------------* * Description: * Does PSOLA epoch stretching or compressing * ********************************************************************** MC ***/ void CBackend::PSOLA_Stretch( float *pInRes, long InSize, float *pOutRes, long OutSize, float *pWindow, long cWindowSize ) { SPDBG_FUNC( "CBackend::PSOLA_Stretch" ); long i, lim; float window, delta, kf; memset( pOutRes, 0, sizeof(float) * OutSize ); lim = MIN(InSize, OutSize ); delta = (float)cWindowSize / (float)lim; kf = 0.5f; pOutRes[0] = pInRes[0]; for( i = 1; i < lim; ++i ) { kf += delta; window = pWindow[(long) kf]; pOutRes[i] += pInRes[i] * window; pOutRes[OutSize - i] += pInRes[InSize - i] * window; } } /* CBackend::PSOLA_Stretch */ /***************************************************************************** * CBackend::PrepareSpeech * *-------------------------* * Description: * ********************************************************************** MC ***/ void CBackend::PrepareSpeech( ISpTTSEngineSite* outputSite ) { SPDBG_FUNC( "CBackend::PrepareSpeech" ); //m_pUnits = pUnits; //m_unitCount = unitCount; //m_CurUnitIndex = 0; m_pOutputSite = outputSite; m_silMode = true; m_durationTarget = 0; m_cOutSamples_Phon = 1; m_cOutEpochs = 0; // Pull model big-bang m_SpeechState = SPEECH_CONTINUE; m_cOutSamples_Total = 0; m_HasSpeech = false; } /* CBackend::PrepareSpeech */ /***************************************************************************** * CBackend::ProsodyMod * *----------------------* * Description: * Calculate the epoch sequence for the synthesized speech * * INPUT: * * OUTPUT: * FIlls 'pOutEpoch', 'pMap', and 'pRevFlag' * Returns new epoch count * ********************************************************************** MC ***/ long CBackend::ProsodyMod( UNITINFO *pCurUnit, long cInEpochs, float durationMpy ) { SPDBG_FUNC( "CBackend::ProsodyMod" ); long iframe, framesize, framesizeOut, j; long cntOut, csamplesOut, cOutEpochs; BOOL fUnvoiced; short fReverse; float totalDuration; float durationIn; // Active accum of IN duration float durationOut; // Active accum of OUT duration aligned to IN domain float freqMpy; BOOL fAdvanceInput; float vibrato; unsigned char *SineWavePtr; float epochFreq; float *pTime; float *pF0; iframe = 0; durationIn = 0.0f; durationOut = 0.0f; csamplesOut = 0; cntOut = 0; cOutEpochs = 0; fReverse = false; pTime = pCurUnit->pTime; pF0 = pCurUnit->pF0; //------------------------------------ // Find total input duration //------------------------------------ totalDuration = 0; for( j = 0; j < cInEpochs; ++j ) { totalDuration += ABS(m_pInEpoch[j]); } /*PredictEpochDist( pCurUnit->duration, pCurUnit->nKnots, m_SampleRate, pTime, pF0 );*/ while( iframe < cInEpochs ) { //----------------------------------------- // Compute output frame length //----------------------------------------- if( m_pInEpoch[iframe] < 0 ) { //------------------------------------------------- // Since we can't change unvoiced pitch, // do not change frame size for unvoiced frames //------------------------------------------------- framesize = (long)((-m_pInEpoch[iframe]) + 0.5f); framesizeOut = framesize; fUnvoiced = true; } else { //--------------------------------------------------- // Modify frame size for voiced epoch // based on epoch frequency //--------------------------------------------------- j = 1; //--------------------------------------------------- // Align to appropriate knot bassed on // current output sample //--------------------------------------------------- while( (j < (long)pCurUnit->nKnots - 1) && (csamplesOut > pTime[j]) ) j++; //--------------------------------------------------- // Calculate exact pitch thru linear interpolation //--------------------------------------------------- epochFreq = LinInterp( pTime[j - 1], (float)csamplesOut, pTime[j], pF0[j - 1], pF0[j] ); SineWavePtr = (unsigned char*)&g_SineWaveTbl[0]; vibrato = (float)(((unsigned char)(*(SineWavePtr + (m_vibrato_Phase1 >> 16)))) - 128); vibrato *= m_VibratoDepth; //--------------------------------------------------- // Scale frame size using in/out ratio //--------------------------------------------------- epochFreq = epochFreq + vibrato; if( epochFreq < MIN_VOICE_PITCH ) { epochFreq = MIN_VOICE_PITCH; } framesize = (long)(m_pInEpoch[iframe] + 0.5f); framesizeOut = (long)(m_SampleRate / epochFreq); vibrato = ((float)256 / ((float)22050 / m_VibratoFreq)) * (float)framesizeOut; // 3 Hz //vibrato = ((float)256 / (float)7350) * (float)framesizeOut; // 3 Hz m_vibrato_Phase1 += (long)(vibrato * (float)65536); m_vibrato_Phase1 &= 0xFFFFFF; //--------------------------------------------------- // @@@@ REMOVED 2x LIMIT //--------------------------------------------------- /*if( framesizeOut > 2*framesize ) { framesizeOut = 2*framesize; } if( framesize > 2*framesizeOut ) { framesizeOut = framesize/2; }*/ freqMpy = (float) framesize / framesizeOut; fUnvoiced = false; } //------------------------------------------- // Generate next output frame //------------------------------------------- fAdvanceInput = false; if( durationOut + (0.5f * framesizeOut/durationMpy) <= durationIn + framesize ) { //----------------------------------------- // If UNvoiced and odd frame, // reverse residual //----------------------------------------- if( fUnvoiced && (cntOut & 1) ) { m_pRevFlag[cOutEpochs] = true; fReverse = true; } else { m_pRevFlag[cOutEpochs] = false; fReverse = false; } ++cntOut; durationOut += framesizeOut/durationMpy; csamplesOut += framesizeOut; m_pOutEpoch[cOutEpochs] = (float)framesizeOut; m_pMap[cOutEpochs] = iframe; cOutEpochs++; } else { fAdvanceInput = true; } //------------------------------------------- // Advance to next input frame //------------------------------------------- if( ((durationOut + (0.5f * framesizeOut/durationMpy)) > (durationIn + framesize)) || //(cntOut >= 3) || @@@@ REMOVED 2x LIMIT //(fReverse == true) || fAdvanceInput ) { durationIn += framesize; ++iframe; cntOut = 0; } } return cOutEpochs; } /* CBackend::ProsodyMod */ /***************************************************************************** * CBackend::LPCFilter * *---------------------* * Description: * LPC filter of order cOrder. It filters the residual signal * pRes, producing output pOutWave. This routine requires that * pOutWave has the true waveform history from [-cOrder,0] and * of course it has to be defined. * ********************************************************************** MC ***/ void CBackend::LPCFilter( float *pCurLPC, float *pCurRes, long len, float gain ) { SPDBG_FUNC( "CBackend::LPCFilter" ); INT t, j; for( t = 0; t < len; t++ ) { m_pHistory[0] = pCurLPC[0] * pCurRes[t]; for( j = m_cOrder; j > 0; j-- ) { m_pHistory[0] -= pCurLPC[j] * m_pHistory[j]; m_pHistory[j] = m_pHistory[j - 1]; } pCurRes[t] = m_pHistory[0] * gain; } } /* CBackend::LPCFilter */ /*void CBackend::LPCFilter( float *pCurLPC, float *pCurRes, long len ) { long t; for( t = 0; t < len; t++ ) { pCurRes[t] = pCurRes[t] * 10; } } */ /***************************************************************************** * CBackend::ResRecons * *---------------------* * Description: * Obtains output prosody modified residual * ********************************************************************** MC ***/ void CBackend::ResRecons( float *pInRes, long InSize, float *pOutRes, long OutSize, float scale ) { SPDBG_FUNC( "CBackend::ResRecons" ); long i, j; if( m_pRevFlag[m_EpochIndex] ) { //---------------------------------------------------- // Process repeated and reversed UNvoiced residual //---------------------------------------------------- for( i = 0, j = OutSize-1; i < OutSize; ++i, --j ) { pOutRes[i] = pInRes[j]; } } else if( InSize == OutSize ) { //---------------------------------------------------- // Unvoiced residual or voiced residual // with no pitch change //---------------------------------------------------- memcpy( pOutRes, pInRes, sizeof(float) *OutSize ); } else { //---------------------------------------------------- // Process voiced residual //---------------------------------------------------- PSOLA_Stretch( pInRes, InSize, pOutRes, OutSize, m_pWindow, m_FFTSize ); } //---------------------------------- // Amplify frame //---------------------------------- if( scale != 1.0f ) { for( i = 0 ; i < OutSize; ++i ) { pOutRes[i] *= scale; } } } /* CBackend::ResRecons */ /***************************************************************************** * CBackend::StartNewUnit * *------------------------* * Description: * Synthesize audio samples for a target unit * * INPUT: * pCurUnit - unit ID, F0, duration, etc. * * OUTPUT: * Sets 'pCurUnit->csamplesOut' with audio length * ********************************************************************** MC ***/ HRESULT CBackend::StartNewUnit( ) { SPDBG_FUNC( "CBackend::StartNewUnit" ); long cframeMax = 0, cInEpochs = 0, i; float totalDuration, durationOut, durationMpy = 0; UNITINFO *pCurUnit; HRESULT hr = S_OK; SPEVENT event; ULONGLONG clientInterest; USHORT volumeVal; // Check for VOLUME change if( m_pOutputSite->GetActions() & SPVES_VOLUME ) { hr = m_pOutputSite->GetVolume( &volumeVal ); if ( SUCCEEDED( hr ) ) { if( volumeVal > SPMAX_VOLUME ) { //--- Clip rate to engine maximum volumeVal = SPMAX_VOLUME; } else if ( volumeVal < SPMIN_VOLUME ) { //--- Clip rate to engine minimum volumeVal = SPMIN_VOLUME; } m_MasterVolume = volumeVal; } } //--------------------------------------- // Delete previous unit //--------------------------------------- CleanUpSynth( ); //--------------------------------------- // Get next phon //--------------------------------------- hr = m_pSrcObj->NextData( (void**)&pCurUnit, &m_SpeechState ); if( m_SpeechState == SPEECH_CONTINUE ) { m_HasSpeech = pCurUnit->hasSpeech; m_pOutputSite->GetEventInterest( &clientInterest ); //------------------------------------------------ // Post SENTENCE event //------------------------------------------------ if( (pCurUnit->flags & SENT_START_FLAG) && (clientInterest & SPFEI(SPEI_SENTENCE_BOUNDARY)) ) { event.elParamType = SPET_LPARAM_IS_UNDEFINED; event.eEventId = SPEI_SENTENCE_BOUNDARY; event.ullAudioStreamOffset = m_cOutSamples_Total * m_BytesPerSample; event.lParam = pCurUnit->sentencePosition; // Input word position event.wParam = pCurUnit->sentenceLen; // Input word length m_pOutputSite->AddEvents( &event, 1 ); } //------------------------------------------------ // Post PHONEME event //------------------------------------------------ if( clientInterest & SPFEI(SPEI_PHONEME) ) { event.elParamType = SPET_LPARAM_IS_UNDEFINED; event.eEventId = SPEI_PHONEME; event.ullAudioStreamOffset = m_cOutSamples_Total * m_BytesPerSample; event.lParam = ((ULONG)pCurUnit->AlloFeatures << 16) + g_IPAToAllo[pCurUnit->AlloID]; event.wParam = ((ULONG)(pCurUnit->duration * 1000.0f) << 16) + g_IPAToAllo[pCurUnit->NextAlloID]; m_pOutputSite->AddEvents( &event, 1 ); } //------------------------------------------------ // Post VISEME event //------------------------------------------------ if( clientInterest & SPFEI(SPEI_VISEME) ) { event.elParamType = SPET_LPARAM_IS_UNDEFINED; event.eEventId = SPEI_VISEME; event.ullAudioStreamOffset = m_cOutSamples_Total * m_BytesPerSample; event.lParam = ((ULONG)pCurUnit->AlloFeatures << 16) + g_AlloToViseme[pCurUnit->AlloID]; event.wParam = ((ULONG)(pCurUnit->duration * 1000.0f) << 16) + g_AlloToViseme[pCurUnit->NextAlloID]; m_pOutputSite->AddEvents( &event, 1 ); } //------------------------------------------------ // Post any bookmark events //------------------------------------------------ if( pCurUnit->pBMObj != NULL ) { CBookmarkList *pBMObj; BOOKMARK_ITEM* pMarker; //------------------------------------------------- // Retrieve marker strings from Bookmark list and // enter into Event list //------------------------------------------------- pBMObj = (CBookmarkList*)pCurUnit->pBMObj; //cMarkerCount = pBMObj->m_BMList.GetCount(); if( clientInterest & SPFEI(SPEI_TTS_BOOKMARK) ) { //--------------------------------------- // Send event for every bookmark in list //--------------------------------------- SPLISTPOS listPos; listPos = pBMObj->m_BMList.GetHeadPosition(); while( listPos ) { pMarker = (BOOKMARK_ITEM*)pBMObj->m_BMList.GetNext( listPos ); event.eEventId = SPEI_TTS_BOOKMARK; event.elParamType = SPET_LPARAM_IS_STRING; event.ullAudioStreamOffset = m_cOutSamples_Total * m_BytesPerSample; //--- Copy in bookmark string - has been NULL terminated in source already... event.lParam = pMarker->pBMItem; // Engine must convert string to long for wParam. event.wParam = _wtol((WCHAR *)pMarker->pBMItem); m_pOutputSite->AddEvents( &event, 1 ); } } //--------------------------------------------- // We don't need this Bookmark list any more //--------------------------------------------- delete pBMObj; pCurUnit->pBMObj = NULL; } pCurUnit->csamplesOut = 0; //****************************************************** // For SIL, fill buffer with zeros... //****************************************************** if( pCurUnit->UnitID == UNIT_SIL ) { //--------------------------------------------- // Calc SIL length //--------------------------------------------- m_durationTarget = (long)(m_SampleRate * pCurUnit->duration); m_cOutSamples_Phon = 0; m_silMode = true; //--------------------------------------------- // Clear LPC filter storage //--------------------------------------------- memset( m_pHistory, 0, sizeof(float)*(m_cOrder+1) ); //-------------------------------- // Success! //-------------------------------- // Debug macro - output unit data... TTSDBG_LOGUNITS; } //****************************************************** // ...otherwise fill buffer with inventory data //****************************************************** else { m_silMode = false; // Get unit data from voice hr = m_pVoiceDataObj->GetUnitData( pCurUnit->UnitID, &m_Synth ); if( SUCCEEDED(hr) ) { durationOut = 0.0f; cInEpochs = m_Synth.cNumEpochs; m_pInEpoch = m_Synth.pEpoch; //cframeMax = PeakValue( m_pInEpoch, cInEpochs ); totalDuration = (float)m_Synth.cNumSamples; //----------------------------------------------- // For debugging: Force duration to unit length //----------------------------------------------- /*float unitDur; unitDur = totalDuration / 22050.0f; if( pCurUnit->duration < unitDur ) { if( pCurUnit->speechRate < 1 ) { pCurUnit->duration = unitDur * pCurUnit->speechRate; } else { pCurUnit->duration = unitDur; } }*/ durationMpy = pCurUnit->duration; cframeMax = (long)pCurUnit->pF0[0]; for( i = 1; i < (long)pCurUnit->nKnots; i++ ) { //----------------------------------------- // Find the longest epoch //----------------------------------------- cframeMax = (long)(MAX(cframeMax,pCurUnit->pF0[i])); } cframeMax *= (long)(durationMpy * MAX_TARGETS_PER_UNIT); durationMpy = (m_SampleRate * durationMpy) / totalDuration; cframeMax += (long)(durationMpy * cInEpochs * MAX_TARGETS_PER_UNIT); // // mplumpe 11/18/97 : added to eliminate chance of crash. // cframeMax *= 2; //--------------------------------------------------- // New epochs adjusted for duration and pitch //--------------------------------------------------- m_pOutEpoch = new float[cframeMax]; if( !m_pOutEpoch ) { //-------------------------------------- // Out of memory! //-------------------------------------- hr = E_OUTOFMEMORY; pCurUnit->csamplesOut = 0; CleanUpSynth( ); } } if( SUCCEEDED(hr) ) { //--------------------------------------------------- // Index back to orig epoch //--------------------------------------------------- m_pMap = new long[cframeMax]; if( !m_pMap ) { //-------------------------------------- // Out of memory! //-------------------------------------- hr = E_OUTOFMEMORY; pCurUnit->csamplesOut = 0; CleanUpSynth( ); } } if( SUCCEEDED(hr) ) { //--------------------------------------------------- // TRUE = reverse residual //--------------------------------------------------- m_pRevFlag = new short[cframeMax]; if( !m_pRevFlag ) { //-------------------------------------- // Out of memory! //-------------------------------------- hr = E_OUTOFMEMORY; pCurUnit->csamplesOut = 0; CleanUpSynth( ); } } if( SUCCEEDED(hr) ) { //--------------------------------------------------------------------- // Compute synthesis epochs and corresponding mapping to analysis // fills in: m_pOutEpoch, m_pMap, m_pRevFlag //--------------------------------------------------------------------- m_cOutEpochs = ProsodyMod( pCurUnit, cInEpochs, durationMpy ); //------------------------------------------------ // Now that actual epoch sizes are known, // calculate total audio sample count // @@@@ NO LONGER NEEDED //------------------------------------------------ pCurUnit->csamplesOut = 0; for( i = 0; i < m_cOutEpochs; i++ ) { pCurUnit->csamplesOut += (long)(ABS(m_pOutEpoch[i])); } m_cOutSamples_Phon = 0; m_EpochIndex = 0; m_durationTarget = (long)(pCurUnit->duration * m_SampleRate); m_pInRes = m_Synth.pRes; m_pLPC = m_Synth.pLPC; m_pSynthTime = pCurUnit->pTime; m_pSynthAmp = pCurUnit->pAmp; m_nKnots = pCurUnit->nKnots; // NOTE: Maybe make log volume? m_UnitVolume = (float)pCurUnit->user_Volume / 100.0f; //------------------------------------------------ // Post WORD event //------------------------------------------------ if( (pCurUnit->flags & WORD_START_FLAG) && (clientInterest & SPFEI(SPEI_WORD_BOUNDARY)) ) { event.elParamType = SPET_LPARAM_IS_UNDEFINED; event.eEventId = SPEI_WORD_BOUNDARY; event.ullAudioStreamOffset = m_cOutSamples_Total * m_BytesPerSample; event.lParam = pCurUnit->srcPosition; // Input word position event.wParam = pCurUnit->srcLen; // Input word length m_pOutputSite->AddEvents( &event, 1 ); } //--- Debug macro - output unit data TTSDBG_LOGUNITS; } } } return hr; } /* CBackend::StartNewUnit */ /***************************************************************************** * CBackend::CleanUpSynth * *------------------------* * Description: * ********************************************************************** MC ***/ void CBackend::CleanUpSynth( ) { SPDBG_FUNC( "CBackend::CleanUpSynth" ); if( m_pOutEpoch ) { delete m_pOutEpoch; m_pOutEpoch = NULL; } if( m_pMap ) { delete m_pMap; m_pMap = NULL; } if( m_pRevFlag ) { delete m_pRevFlag; m_pRevFlag = NULL; } // NOTE: make object? FreeSynth( &m_Synth ); } /* CBackend::CleanUpSynth */ /***************************************************************************** * CBackend::RenderFrame * *-----------------------* * Description: * This this the central synthesis loop. Keep filling output audio * buffer until buffer frame is full or speech is done. To render * continous speech, get each unit one at a time from upstream buffer. * ********************************************************************** MC ***/ HRESULT CBackend::RenderFrame( ) { SPDBG_FUNC( "CBackend::RenderFrame" ); long InSize, OutSize; long iframe; float *pCurInRes, *pCurOutRes; long i, j; float ampMpy; HRESULT hr = S_OK; m_cOutSamples_Frame = 0; do { OutSize = 0; if( m_silMode ) { //------------------------------- // Silence mode //------------------------------- if( m_cOutSamples_Phon >= m_durationTarget ) { //--------------------------- // Get next unit //--------------------------- hr = StartNewUnit( ); if (FAILED(hr)) { //----------------------------------- // Try to end it gracefully... //----------------------------------- m_SpeechState = SPEECH_DONE; } TTSDBG_LOGSILEPOCH; } else { //--------------------------- // Continue with current SIL //--------------------------- m_pSpeechBuf[m_cOutSamples_Frame] = 0; OutSize = 1; } } else { if( m_EpochIndex < m_cOutEpochs ) { //-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // // Continue with current phon // //-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ //------------------------------------ // Find current input residual //------------------------------------ iframe = m_pMap[m_EpochIndex]; pCurInRes = m_pInRes; for( i = 0; i < iframe; i++) { pCurInRes += (long) ABS(m_pInEpoch[i]); } pCurOutRes = m_pSpeechBuf + m_cOutSamples_Frame; InSize = (long)(ABS(m_pInEpoch[iframe])); OutSize = (long)(ABS(m_pOutEpoch[m_EpochIndex])); j = 1; while( (j < m_nKnots - 1) && (m_cOutSamples_Phon > m_pSynthTime[j]) ) { j++; } ampMpy = LinInterp( m_pSynthTime[j - 1], (float)m_cOutSamples_Phon, m_pSynthTime[j], m_pSynthAmp[j - 1], m_pSynthAmp[j] ); //ampMpy = 1; //-------------------------------------------- // Do stretching of residuals //-------------------------------------------- ResRecons( pCurInRes, InSize, pCurOutRes, OutSize, ampMpy ); //-------------------------------------------- // Do LPC reconstruction //-------------------------------------------- float *pCurLPC; float totalGain; totalGain = ExpConverter( ((float)m_MasterVolume / (float)SPMAX_VOLUME), m_linearScale ) * ExpConverter( m_UnitVolume, m_linearScale ); pCurLPC = m_pLPC + m_pMap[m_EpochIndex] * (1 + m_cOrder); pCurLPC[0] = 1.0f; LPCFilter( pCurLPC, &m_pSpeechBuf[m_cOutSamples_Frame], OutSize, totalGain ); m_EpochIndex++; } else { //-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // // Get next phon // //-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ hr = StartNewUnit( ); if (FAILED(hr)) { //----------------------------------- // Try to end it gracefully... //----------------------------------- m_SpeechState = SPEECH_DONE; } TTSDBG_LOGSILEPOCH; } } m_cOutSamples_Frame += OutSize; m_cOutSamples_Phon += OutSize; m_cOutSamples_Total += OutSize; TTSDBG_LOGEPOCHS; } while( (m_cOutSamples_Frame < SPEECH_FRAME_SIZE) && (m_SpeechState == SPEECH_CONTINUE) ); if( SUCCEEDED(hr) ) { //---------------------------------------------- // Convert buffer from FLOAT to SHORT //---------------------------------------------- if( m_pReverb ) { //--------------------------------- // Add REVERB //--------------------------------- m_pReverb->Reverb_Process( m_pSpeechBuf, m_cOutSamples_Frame, 1.0f ); } else { CvtToShort( m_pSpeechBuf, m_cOutSamples_Frame, m_StereoOut, 1.0f ); } //--- Debug Macro - output wave data to stream TTSDBG_LOGWAVE; } if( SUCCEEDED( hr ) ) { //------------------------------------ // Send this buffer to SAPI site //------------------------------------ DWORD cbWritten; //------------------------------------------------------------------------------------ // This was my lame hack to avoid sending buffers when nothing was spoken. // It was causing problems (among others) since StartNewUnit() was still sending // events - with no corresponding audio buffer! // // This was too simple of a scheme. Disable this feature for now... // ...until I come up with something more robust. (MC) //------------------------------------------------------------------------------------ //if( m_HasSpeech ) { hr = m_pOutputSite->Write( (void*)m_pSpeechBuf, m_cOutSamples_Frame * m_BytesPerSample, &cbWritten ); if( FAILED( hr ) ) { //---------------------------------------- // Abort! Unable to write audio data //---------------------------------------- m_SpeechState = SPEECH_DONE; } } } //------------------------------------ // Return render state //------------------------------------ return hr; } /* CBackend::RenderFrame */