diff --git a/ai/BUILD.gn b/ai/BUILD.gn new file mode 100644 index 0000000000000000000000000000000000000000..336422cf4058542b197cfb6f5f922708b780acdb --- /dev/null +++ b/ai/BUILD.gn @@ -0,0 +1,20 @@ +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import("//build/lite/config/component/lite_component.gni") + +lite_component("ai") { + features = [ + "asr:asr", + "cv:cv", + ] +} diff --git a/ai/asr/BUILD.gn b/ai/asr/BUILD.gn new file mode 100644 index 0000000000000000000000000000000000000000..183052122cd8053fa5b22d974cba18b024156783 --- /dev/null +++ b/ai/asr/BUILD.gn @@ -0,0 +1,17 @@ +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import("//build/lite/config/component/lite_component.gni") + +lite_component("asr") { + features = [ "keyword_spotting:kws" ] +} diff --git a/ai/asr/keyword_spotting/BUILD.gn b/ai/asr/keyword_spotting/BUILD.gn new file mode 100644 index 0000000000000000000000000000000000000000..7dcf20e0e6631d8a1f4dbaead68803d31b49f956 --- /dev/null +++ b/ai/asr/keyword_spotting/BUILD.gn @@ -0,0 +1,52 @@ +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import("//build/lite/config/component/lite_component.gni") + +executable("kws_app") { + sources = [ + "audio_cache.cpp", + "kws_app.cpp", + "kws_manager.cpp", + ] + cflags = [ "-Wall" ] + cflags_cc = cflags + include_dirs = [ + "//foundation/ai/engine/services/common/platform/os_wrapper/audio_loader/include", + "//foundation/ai/engine/services/common/platform/os_wrapper/audio_loader/include/codec", + "//foundation/ai/engine/interfaces/kits/asr/keyword_spotting", + "//foundation/ai/engine/interfaces/kits", + "//foundation/multimedia/interfaces/kits/audio_lite", + "//foundation/multimedia/utils/include", + "//foundation/ai/engine/services/common/utils", + "//foundation/ai/engine/services/common/platform/utils", + "//foundation/ai/engine/services/common/platform/os_wrapper/utils", + "//third_party/bounds_checking_function/include", + ] + ldflags = [ + "-lstdc++", + "-lpthread", + "-lcodec", + ] + deps = [ + "//foundation/ai/engine/services/client/algorithm_sdk/asr:asr", + "//foundation/ai/engine/services/common/platform/os_wrapper/audio_loader:audio_loader", + "//foundation/multimedia/audio_lite/frameworks:audio_capturer_lite", + "//foundation/multimedia/utils/lite:media_common", + "//third_party/bounds_checking_function:libsec_static", + ] + output_dir = "$root_out_dir/dev_tools" +} + +lite_component("kws") { + features = [ ":kws_app" ] +} diff --git a/ai/asr/keyword_spotting/audio_cache.cpp b/ai/asr/keyword_spotting/audio_cache.cpp new file mode 100644 index 0000000000000000000000000000000000000000..895906dffa1e2d020eaa60cee685b04952a8e710 --- /dev/null +++ b/ai/asr/keyword_spotting/audio_cache.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "audio_cache.h" + +#include + +#include "securec.h" + +namespace KWS { +const int32_t MAX_BUFFER_SIZE = 36000; +const int32_t MIN_BUFFER_SIZE = 8000; + +AudioCache::AudioCache() + : maxBufferSize_(0), left_(0), right_(0), buffer_(nullptr), prepared_(false) +{ +} + +AudioCache::~AudioCache() +{ + if (prepared_ && buffer_ != nullptr) { + delete[] buffer_; + buffer_ = nullptr; + } + prepared_ = false; + maxBufferSize_ = 0; + left_ = 0; + right_ = 0; +} + +bool AudioCache::Init(int32_t maxSize) +{ + if (prepared_) { + printf("[AudioCache]Cache has ready init.\n"); + return false; + } + if (maxSize > MAX_BUFFER_SIZE || maxSize < MIN_BUFFER_SIZE) { + printf("[AudioCache]maxSize out of range, init failed.\n"); + return false; + } + maxBufferSize_ = maxSize; + buffer_ = new (std::nothrow) uint8_t[maxBufferSize_]; + if (buffer_ == nullptr) { + printf("[AudioCache]Fail to allocate buffer for given size.\n"); + return false; + } + errno_t ret = memset_s(buffer_, maxBufferSize_, 0x00, maxBufferSize_); + if (ret != EOK) { + printf("[AudioCache]Cache buffer init failed.\n"); + return false; + } + prepared_ = true; + return true; +} + +bool AudioCache::AppendBuffer(int32_t bufferSize, const uint8_t *buffer) +{ + if (!prepared_) { + return false; + } + if (bufferSize + right_ >= maxBufferSize_) { + left_ = 0; + right_ = 0; + } + size_t remainLength = maxBufferSize_ - right_; + size_t length = (bufferSize > remainLength) ? remainLength : bufferSize; + size_t copySize = sizeof(uint8_t) * length; + errno_t ret = memcpy_s(&buffer_[right_], copySize, buffer, copySize); + if (ret != EOK) { + printf("[AudioCache]AppendBuffer failed.\n"); + return false; + } + right_ += length; + return true; +} + +size_t AudioCache::GetCapturedBuffer(uintptr_t &samplePtr) +{ + if (right_ - left_ <= 0 || buffer_ == nullptr) { + return 0; + } + samplePtr = reinterpret_cast(buffer_ + left_); + size_t length = right_ - left_; + left_ = right_; + return length; +} +} // namespace KWS \ No newline at end of file diff --git a/ai/asr/keyword_spotting/audio_cache.h b/ai/asr/keyword_spotting/audio_cache.h new file mode 100644 index 0000000000000000000000000000000000000000..6e4f1a68d65cbd4174f78d573542fd1036402f48 --- /dev/null +++ b/ai/asr/keyword_spotting/audio_cache.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AUDIO_CACHE_H +#define AUDIO_CACHE_H + +#include +#include + +namespace KWS { +class AudioCache { +public: + AudioCache(); + virtual ~AudioCache(); + bool Init(int32_t maxSize); + size_t GetCapturedBuffer(uintptr_t &samplePtr); + bool AppendBuffer(int32_t bufferSize, const uint8_t *buffer); + +private: + int32_t maxBufferSize_; + int32_t left_; + int32_t right_; + uint8_t *buffer_; + bool prepared_; +}; +} // namespace KWS +#endif // AUDIO_CACHE_H \ No newline at end of file diff --git a/ai/asr/keyword_spotting/kws_app.cpp b/ai/asr/keyword_spotting/kws_app.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d08198ea93329fee91246a9e298f300a1e835e8c --- /dev/null +++ b/ai/asr/keyword_spotting/kws_app.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "kws_manager.h" + +using namespace std; +using namespace KWS; + +static void SampleHelp() +{ + printf("****************************************\n"); + printf("Select the behavior of KWS app.\n"); + printf("s: AudioCap (Press q to quit)\n"); + printf("q: quit the sample.\n"); + printf("****************************************\n"); +} + +int main() +{ + printf("Keyword spotting started.\n"); + SampleHelp(); + + shared_ptr kwsMgr = make_shared(AUDIO_SAMPLE_RATE, AUDIO_CODEC_BITRATE); + if (kwsMgr == nullptr) { + printf("Keyword spotting failed to allocate KWSManager.\n"); + return -1; + } + if (!kwsMgr->Prepare()) { + printf("Keyword spotting failed to prepare KWSManager.\n"); + return -1; + } + char input = ' '; + while (cin >> input) { + switch (input) { + case 's': + kwsMgr->Start(); + break; + case 'q': + kwsMgr->Stop(); + printf("Keyword spotting Terminated.\n"); + return 0; + default: + SampleHelp(); + break; + } + } + return 0; +} \ No newline at end of file diff --git a/ai/asr/keyword_spotting/kws_manager.cpp b/ai/asr/keyword_spotting/kws_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..de4e5dca9e6692808930680a3af212ca51b89465 --- /dev/null +++ b/ai/asr/keyword_spotting/kws_manager.cpp @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "kws_manager.h" + +using namespace OHOS::AI; +using namespace OHOS::Audio; +using namespace OHOS::Media; + +namespace KWS { +namespace { + const int32_t CONFIDENCE = 2662; + const int32_t NUM_SLIDE_WINDOW = 10; + const int32_t MAX_CACHE_SIZE = 16000; + const int32_t WINDOW_SIZE = 480; // times of per window: 30ms + const int32_t STRIDE_SIZE = 320; // times of per stride: 20ms + const int32_t SUCCESS = 0; + const int32_t CHANNEL_COUNT = 1; + + using Capturer = std::shared_ptr; + using Cache = std::shared_ptr; + using Plugin = std::shared_ptr; + + // Can not create KwsManager twice + std::shared_ptr g_producer = nullptr; + std::shared_ptr g_consumer = nullptr; + const auto DELETE_ARRAY = [](uint8_t *array) { + if (array != nullptr) { + delete[] array; + } + }; +} + +class MyKwsCallback : public KWSCallback { +public: + MyKwsCallback() {} + ~MyKwsCallback() {} + + void OnError(int32_t errorCode) + { + printf("Executing error, error code: %d\n", errorCode); + } + + void OnResult(const Array &result) + { + if (result.size != WORD_CONTENT.size()) { + return; + } + for (size_t i = 0; i < result.size; ++i) { + if (result.data[i] > CONFIDENCE) { + printf("[%s]\n", WORD_CONTENT[i].c_str()); + } + } + } +}; + +void ThreadTask::AudioProducer(KwsManager *kwsManager) +{ + kwsManager->ProduceSamples(); +} + +void ThreadTask::AudioConsumer(KwsManager *kwsManager) +{ + kwsManager->ConsumeSamples(); +} + +KwsManager::KwsManager(int32_t sampleRate, int32_t bitRate) + : sampleRate_(sampleRate), bitRate_(bitRate) +{ + status_ = IDLE; +} + +KwsManager::~KwsManager() +{ + OnStop(); + status_ = IDLE; +} + +void KwsManager::ProduceSamples() +{ + printf("[KwsManager]ProduceSamples start\n"); + if (capturer_ == nullptr || cache_ == nullptr || aacHandler_ == -1) { + printf("[KwsManager]Produce AudioSample failed for nullptr\n"); + return; + } + int32_t readLen = 0; + int32_t retCode = SUCCESS; + size_t frameSize = capturer_->GetFrameCount() * sizeof(uint16_t); + if (frameSize <= 0) { + printf("[KwsManager]Capturer get frame count failed.\n"); + return; + } + CoderStream aacStream = { + .buffer = new (std::nothrow) uint8_t[frameSize], + .size = frameSize + }; + CoderStream pcmStream = { + .buffer = new (std::nothrow) uint8_t[frameSize], + .size = frameSize + }; + if (aacStream.buffer == nullptr || pcmStream.buffer == nullptr) { + printf("[KwsManager]Allocate buffer for aacStream and pcmStream failed.\n"); + DELETE_ARRAY(aacStream.buffer); + DELETE_ARRAY(pcmStream.buffer); + return; + } + while (status_ == RUNNING) { + { + std::lock_guard lock(mutex_); + if (capturer_ == nullptr) { + break; + } + readLen = capturer_->Read(aacStream.buffer, frameSize, false); + } + if (readLen <= 0 || readLen > static_cast(frameSize)) { + continue; + } + aacStream.size = readLen; + retCode = AudioWrapper::GetInstance().Convert(aacHandler_, aacStream, pcmStream); + if (retCode != SUCCESS) { + continue; + } + { + std::lock_guard lock(mutex_); + if (cache_ == nullptr) { + break; + } + if (!cache_->AppendBuffer(pcmStream.size, pcmStream.buffer)) { + printf("[KwsManager]Fail to append pcm into cache.\n"); + } + } + } + DELETE_ARRAY(aacStream.buffer); + DELETE_ARRAY(pcmStream.buffer); +} + +void KwsManager::ConsumeSamples() +{ + uintptr_t sampleAddr = 0; + size_t sampleSize = 0; + int32_t retCode = SUCCESS; + while (status_ == RUNNING) { + { + std::lock_guard lock(mutex_); + if (cache_ == nullptr) { + printf("[KwsManager]cache_ is nullptr\n"); + break; + } + sampleSize = cache_->GetCapturedBuffer(sampleAddr); + } + if (sampleSize == 0 || sampleAddr == 0) { + continue; + } + Array input = { + .data = (int16_t *)sampleAddr, + .size = sampleSize >> 1 + }; + { + std::lock_guard lock(mutex_); + if (plugin_ == nullptr) { + printf("[KwsManager]cache_ is nullptr\n"); + break; + } + if ((retCode = plugin_->SyncExecute(input)) != SUCCESS) { + printf("[KwsManager]SyncExecute KWS failed with retcode = [%d]\n", retCode); + continue; + } + } + } +} + +bool KwsManager::Prepare() +{ + printf("[KwsManager]Prepare.\n"); + if (status_ != IDLE) { + printf("[KwsManager]Already prepared.\n"); + return false; + } + if (!PreparedAudioCapturer()) { + printf("[KwsManager]Fail to prepare AudioCapturer!\n"); + OnStop(); + return false; + } + if (!PreparedAudioWrapper()) { + printf("[KwsManager]Fail to prepare AudioWrapper!\n"); + OnStop(); + return false; + } + if (!PreparedInference()) { + printf("[KwsManager]Fail to prepare Inference!\n"); + OnStop(); + return false; + } + status_ = PREPARED; + return true; +} + +void KwsManager::Start() +{ + printf("[KwsManager]Start.\n"); + if (status_ == RUNNING) { + printf("[KwsManager]Already running.\n"); + return; + } + if (status_ != PREPARED && !Prepare()) { + printf("[KwsManager]Fail to prepare KwsManager!\n"); + return; + } + OnStart(); +} + +void KwsManager::Stop() +{ + printf("[KwsManager]Stop.\n"); + if (status_ == IDLE) { + printf("[KwsManager]Is already stopped.\n"); + return; + } + std::lock_guard lock(mutex_); + status_ = IDLE; + OnStop(); +} + +void KwsManager::OnStart() +{ + if (!capturer_->Start()) { + printf("[KwsManager]Fail to start audioCapturer\n"); + OnStop(); + return; + } + if (aacHandler_ == -1) { + printf("[KwsManager]Fail to start producer for the illegal aac-to-pcm handler\n"); + OnStop(); + return; + } + if (cache_ == nullptr) { + printf("[KwsManager]Fail to start producer for the nullptr cache\n"); + OnStop(); + return; + } + if (plugin_ == nullptr) { + printf("[KwsManager]Fail to start producer for the nullptr plugin\n"); + OnStop(); + return; + } + status_ = RUNNING; + g_producer = std::make_shared(ThreadTask::AudioProducer, this); + g_consumer = std::make_shared(ThreadTask::AudioConsumer, this); +} + +void KwsManager::StopAudioCapturer() +{ + printf("[KwsManager]StopAudioCapturer\n"); + if (capturer_ != nullptr) { + capturer_->Stop(); + capturer_ = nullptr; + } +} + +void KwsManager::StopAudioWrapper() +{ + printf("[KwsManager]StopAudioWrapper\n"); + AudioWrapper::GetInstance().Deinit(aacHandler_); +} + +void KwsManager::StopInference() +{ + printf("[KwsManager]StopInference\n"); + if (plugin_ != nullptr) { + int32_t ret = plugin_->Destroy(); + if (ret != SUCCESS) { + printf("[KwsManager]plugin_ destroy failed.\n"); + } + plugin_ = nullptr; + } +} + +void KwsManager::OnStop() +{ + if (capturer_ != nullptr) { + StopAudioCapturer(); + } + StopAudioWrapper(); + if (plugin_ != nullptr) { + StopInference(); + } + if (g_producer != nullptr) { + g_producer->join(); + } + if (g_consumer != nullptr) { + g_consumer->join(); + } +} + +bool KwsManager::PreparedAudioCapturer() +{ + // Set audio wrapper mode before build audio capturer + AudioWrapper::GetInstance().SetCodecMode(false); + if (capturer_ != nullptr) { + printf("[KwsManager]Stop created AudioCapturer at first\n"); + StopAudioCapturer(); + } + capturer_ = std::make_shared(); + if (capturer_ == nullptr || !ConfiguredAudioCapturer()) { + printf("[KwsManager]Fail to create AudioCapturer.\n"); + OnStop(); + return false; + } + return true; +} + +bool KwsManager::PreparedAudioWrapper() +{ + cache_ = std::make_shared(); + if (cache_ == nullptr) { + printf("[KwsManager]Failed to create AudioCache\n"); + return false; + } + if (!cache_->Init(MAX_CACHE_SIZE)) { + printf("[KwsManager]Failed to init AudioCache\n"); + return false; + } + if (!ConfiguredAudioWrapper()) { + printf("[KwsManager]Failed to prepared AudioWrapper.\n"); + OnStop(); + return false; + } + return true; +} + +bool KwsManager::PreparedInference() +{ + if (capturer_ == nullptr) { + printf("[KwsManager]Only load plugin after AudioCapturer ready.\n"); + return false; + } + if (plugin_ != nullptr) { + printf("[KwsManager]Stop created Inference Plugin at first.\n"); + StopInference(); + } + plugin_ = std::make_shared(); + if (plugin_ == nullptr) { + printf("[KwsManager]Failed to create InferencePlugin.\n"); + return false; + } + if (plugin_->Create() != SUCCESS) { + printf("[KwsManager]Failed to create KWSSDK.\n"); + return false; + } + std::shared_ptr callback = std::make_shared(); + if (callback == nullptr) { + printf("[KwsManager]Create callback failed.\n"); + return false; + } + plugin_->SetCallback(callback); + return true; +} + +bool KwsManager::ConfiguredAudioCapturer() +{ + AudioCapturerInfo audioConfig; + audioConfig.inputSource = AUDIO_MIC; + audioConfig.audioFormat = AAC_LC; + audioConfig.sampleRate = AUDIO_SAMPLE_RATE; + audioConfig.channelCount = CHANNEL_COUNT; + audioConfig.bitRate = AUDIO_CODEC_BITRATE; + audioConfig.streamType = TYPE_MEDIA; + audioConfig.bitWidth = BIT_WIDTH_16; + int32_t ret = capturer_->SetCapturerInfo(audioConfig); + if (ret != SUCCESS) { + printf("[KwsManager]ConfiguredAudioCapturer fail with ret = 0x%.8x\n", ret); + return false; + } + return true; +} + +bool KwsManager::ConfiguredAudioWrapper() +{ + ConvertType typo = CONVERT_AAC_TO_PCM; + CoderConfig codecConfig; + codecConfig.audioFormat = AAC_LC; + codecConfig.bitRate = bitRate_; + codecConfig.sampleRate = sampleRate_; + codecConfig.channelCount = CHANNEL_COUNT; + codecConfig.bitWidth = BIT_WIDTH_16; + if (AudioWrapper::GetInstance().Init(typo, aacHandler_) != SUCCESS) { + return false; + } + if (AudioWrapper::GetInstance().SetConfig(aacHandler_, codecConfig) != SUCCESS) { + return false; + } + if (AudioWrapper::GetInstance().StartCodec(aacHandler_) != SUCCESS) { + return false; + } + return true; +} +} \ No newline at end of file diff --git a/ai/asr/keyword_spotting/kws_manager.h b/ai/asr/keyword_spotting/kws_manager.h new file mode 100644 index 0000000000000000000000000000000000000000..17c9b897f26d9cf71ed71947eacd75a084fa39d4 --- /dev/null +++ b/ai/asr/keyword_spotting/kws_manager.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef KWS_MANAGER_H +#define KWS_MANAGER_H + +#include +#include +#include +#include + +#include "ai_datatype.h" +#include "audio_cache.h" +#include "audio_capturer.h" +#include "audio_wrapper.h" +#include "kws_callback.h" +#include "kws_sdk.h" +#include "media_errors.h" +#include "media_info.h" + +namespace KWS { +const int32_t AUDIO_SAMPLE_RATE = 16000; // 16kHz +const int32_t AUDIO_CODEC_BITRATE = 32000; // 32kHz + +const std::vector WORD_CONTENT = { + "Hi Xiaowen", + "Nihao Wenwen", + "Unknown" +}; + +enum KwsStatus { + IDLE = 1000, + PREPARED, + RUNNING, +}; + +class KwsManager { +public: + KwsManager(int32_t sampleRate, int32_t bitRate); + ~KwsManager(); + void Start(); + void Stop(); + bool Prepare(); + void ProduceSamples(); + void ConsumeSamples(); + +private: + void OnStart(); + void OnStop(); + bool PreparedAudioCapturer(); + bool PreparedAudioWrapper(); + bool PreparedInference(); + bool ConfiguredAudioCapturer(); + bool ConfiguredAudioWrapper(); + void StopAudioCapturer(); + void StopAudioWrapper(); + void StopInference(); + +private: + std::shared_ptr cache_ = nullptr; + std::shared_ptr plugin_ = nullptr; + std::shared_ptr capturer_ = nullptr; + int32_t sampleRate_ = AUDIO_SAMPLE_RATE; + int32_t bitRate_ = AUDIO_CODEC_BITRATE; + KwsStatus status_ = IDLE; + intptr_t aacHandler_ = -1; + std::mutex mutex_; +}; + +class ThreadTask { +public: + static void AudioProducer(KwsManager *kwsManager); + static void AudioConsumer(KwsManager *kwsManager); +}; +} // namespace KWS +#endif \ No newline at end of file diff --git a/ai/cv/BUILD.gn b/ai/cv/BUILD.gn new file mode 100644 index 0000000000000000000000000000000000000000..35b33bb3ad7ec30ad30f8a4531a322cf8fb99fce --- /dev/null +++ b/ai/cv/BUILD.gn @@ -0,0 +1,17 @@ +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import("//build/lite/config/component/lite_component.gni") + +lite_component("cv") { + features = [ "image_classification:ic" ] +} diff --git a/ai/cv/image_classification/BUILD.gn b/ai/cv/image_classification/BUILD.gn new file mode 100644 index 0000000000000000000000000000000000000000..e2e58f97977ed4ff55092aa60f7220e1f9bd8831 --- /dev/null +++ b/ai/cv/image_classification/BUILD.gn @@ -0,0 +1,52 @@ +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import("//build/lite/config/component/lite_component.gni") + +copy("demo_image_for_test") { + sources = [ "resources/image_classification_demo.jpg" ] + outputs = [ "$root_out_dir/data/image_classification_demo.jpg" ] +} + +executable("ic_app") { + sources = [ + "camera_manager.cpp", + "image_classification_sample.cpp", + "picture_utils.cpp", + "resize_computer.cpp", + ] + cflags = [ "-Wall" ] + cflags_cc = cflags + include_dirs = [ + "//third_party/libjpeg", + "//foundation/ai/engine/interfaces/kits", + "//foundation/ai/engine/interfaces/kits/cv/image_classification", + "//foundation/ai/engine/services/client/algorithm_sdk/cv/image_classification/include", + "//foundation/multimedia/media_lite/interfaces/kits/recorder_lite", + ] + + ldflags = [ "-lstdc++" ] + ldflags += [ "-lpthread" ] + ldflags += [ "-Wl,-rpath-link=$ohos_root_path/$root_out_dir" ] + + deps = [ + "//foundation/ai/engine/services/client/algorithm_sdk/cv/image_classification:image_classification_sdk", + "//foundation/multimedia/camera_lite/frameworks:camera_lite", + "//foundation/multimedia/media_lite/frameworks/recorder_lite:recorder_lite", + "//third_party/libjpeg", + ] + output_dir = "$root_out_dir/dev_tools" +} + +lite_component("ic") { + features = [ ":ic_app" ] +} diff --git a/ai/cv/image_classification/camera_manager.cpp b/ai/cv/image_classification/camera_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8c5557da335a247a12c87e0b8a2e212da809a857 --- /dev/null +++ b/ai/cv/image_classification/camera_manager.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "camera_manager.h" + +using namespace std; +using namespace OHOS; +using namespace OHOS::Media; + +namespace IC { +const int QFACTOR = 90; +std::mutex g_mutex; +std::condition_variable g_cv; + +static void SampleSaveCapture(const char *p, uint32_t size) +{ + printf("Start saving picture.\n"); + ofstream pic(CAMERA_SAVE_PATH, ofstream::out | ofstream::trunc); + if (pic.is_open()) { + printf("Write %d bytes.\n", size); + pic.write(p, size); + pic.close(); + printf("Saving picture end.\n"); + } +} + +void SampleSurfaceListener::SetSurface(Surface *surface) +{ + surface_ = surface; +} + +void SampleFrameStateCallback::OnFrameFinished(Camera &camera, FrameConfig &fc, FrameResult &result) +{ + std::unique_lock lock(g_mutex); + printf("Receive frame complete inform.\n"); + if (fc.GetFrameConfigType() == FRAME_CONFIG_CAPTURE) { + printf("Capture frame received.\n"); + list surfaceList = fc.GetSurfaces(); + for (Surface *surface : surfaceList) { + SurfaceBuffer *buffer = surface->AcquireBuffer(); + if (buffer != nullptr) { + char *virtAddr = static_cast(buffer->GetVirAddr()); + if (virtAddr != nullptr) { + SampleSaveCapture(virtAddr, buffer->GetSize()); + } + surface->ReleaseBuffer(buffer); + } + delete surface; + } + } else if (fc.GetFrameConfigType() == FRAME_CONFIG_PREVIEW) { + printf("Preview frame received.\n"); + list surfaceList = fc.GetSurfaces(); + for (Surface *surface : surfaceList) { + printf("surface release.\n"); + delete surface; + } + } + delete &fc; + g_cv.notify_all(); +} + +void SampleSurfaceListener::OnBufferAvailable() +{ + if (surface_ == nullptr) { + return; + } + SurfaceBuffer *buffer = surface_->AcquireBuffer(); + if (buffer != nullptr) { + surface_->ReleaseBuffer(buffer); + } +} + +SampleCameraStateMng::SampleCameraStateMng(EventHandler &eventHdlr) : eventHdlr_(eventHdlr) +{ +} + +SampleCameraStateMng::~SampleCameraStateMng() +{ +} + +void SampleCameraStateMng::OnCreated(Camera &c) +{ + std::unique_lock lock(g_mutex); + printf("Sample recv OnCreate camera.\n"); + auto config = CameraConfig::CreateCameraConfig(); + if (config != nullptr) { + config->SetFrameStateCallback(&fsCb_, &eventHdlr_); + } + c.Configure(*config); + cam_ = &c; + g_cv.notify_all(); +} + +void SampleCameraStateMng::OnCreateFailed(const string cameraId, int32_t errorCode) +{ + printf("Sample camera create failed. Error code: %d.\n", errorCode); +} + +void SampleCameraStateMng::OnReleased(Camera &c) +{ + printf("Sample camera OnReleased.\n"); +} + +void SampleCameraStateMng::Capture() +{ + if (cam_ == nullptr) { + printf("Camera is not ready.\n"); + return; + } + FrameConfig *fc = new (std::nothrow) FrameConfig(FRAME_CONFIG_CAPTURE); + if (fc == nullptr) { + printf("Create FrameConfig failed.\n"); + return; + } + Surface *surface = Surface::CreateSurface(); + if (surface == nullptr) { + delete fc; + printf("CreateSurface failed.\n"); + return; + } + surface->SetWidthAndHeight(CAMERA_PIC_WIDTH, CAMERA_PIC_HEIGHT); + surface->SetUsage(BUFFER_CONSUMER_USAGE_HARDWARE); + fc->AddSurface(*surface); + fc->SetParameter(PARAM_KEY_IMAGE_ENCODE_QFACTOR, QFACTOR); + cam_->TriggerSingleCapture(*fc); +} +} // namespace IC \ No newline at end of file diff --git a/ai/cv/image_classification/camera_manager.h b/ai/cv/image_classification/camera_manager.h new file mode 100644 index 0000000000000000000000000000000000000000..664a71645032547d210e1fa4a455b697c1b39c7c --- /dev/null +++ b/ai/cv/image_classification/camera_manager.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CAMERA_MANAGER_H +#define CAMERA_MANAGER_H + +#include "camera_kit.h" +#include "recorder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace IC { +// Settings of camera +const int32_t CAMERA_PIC_WIDTH = 1920; +const int32_t CAMERA_PIC_HEIGHT = 1080; +const std::string CAMERA_SAVE_PATH = "/storage/Capture.jpg"; + +// condition_variable for camera +extern std::mutex g_mutex; +extern std::condition_variable g_cv; + +class SampleFrameStateCallback : public OHOS::Media::FrameStateCallback { +public: + void OnFrameFinished(OHOS::Media::Camera &camera, + OHOS::Media::FrameConfig &fc, + OHOS::Media::FrameResult &result) override; +}; + +class SampleSurfaceListener : public OHOS::IBufferConsumerListener { +public: + void SetSurface(OHOS::Media::Surface *surface); + void OnBufferAvailable() override; + +private: + OHOS::Media::Surface *surface_ = nullptr; +}; + +class SampleCameraDeivceCallback : public OHOS::Media::CameraDeviceCallback { +}; + +class SampleCameraStateMng : public OHOS::Media::CameraStateCallback { +public: + SampleCameraStateMng() = delete; + explicit SampleCameraStateMng(OHOS::Media::EventHandler &eventHdlr); + ~SampleCameraStateMng(); + void OnCreated(OHOS::Media::Camera &c) override; + void OnCreateFailed(const std::string cameraId, int32_t errorCode) override; + void OnReleased(OHOS::Media::Camera &c) override; + void Capture(); +private: + OHOS::Media::EventHandler &eventHdlr_; + OHOS::Media::Camera *cam_ = nullptr; + SampleSurfaceListener listener_; + SampleFrameStateCallback fsCb_; + OHOS::Media::FrameConfig *fc_ = nullptr; +}; +} // namespace IC +#endif \ No newline at end of file diff --git a/ai/cv/image_classification/image_classification_sample.cpp b/ai/cv/image_classification/image_classification_sample.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d24097e82ce3ed5ba926d05d2b79c94f2b406a79 --- /dev/null +++ b/ai/cv/image_classification/image_classification_sample.cpp @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "camera_manager.h" +#include "ic_callback.h" +#include "ic_constants.h" +#include "ic_retcode.h" +#include "ic_sdk.h" +#include "picture_utils.h" + +using namespace std; +using namespace IC; +using namespace OHOS; +using namespace OHOS::AI; +using namespace OHOS::Media; + +namespace { +const auto DELETE_PIC_BUFFER = [](uint8_t *&buffer) { + if (buffer != nullptr) { + delete[] buffer; + buffer = nullptr; + } +}; +} + +class MyIcCallback : public IcCallback { +public: + MyIcCallback() {} + ~MyIcCallback() {} + + void OnError(const IcRetCode errorCode) + { + printf("Executing error, error code: %d\n", errorCode); + } + + void OnResult(const IcOutput &result) + { + if (result.size % 2 == 0) { + int numClass = result.size / 2; + for (size_t i = 0; i < numClass; ++i) { + printf("class index [%d] score: %d\n", result.data[i], result.data[i + numClass]); + } + } + } +}; + +static int CameraCapture() +{ + printf("Camera sample begin.\n"); + CameraKit *camKit = CameraKit::GetInstance(); + if (camKit == nullptr) { + printf("Can not get CameraKit instance\n"); + return IC_RETCODE_FAILURE; + } + list camList = camKit->GetCameraIds(); + string camId; + for (auto &cam : camList) { + printf("camera name: %s\n", cam.c_str()); + const CameraAbility *ability = camKit->GetCameraAbility(cam); + list sizeList = ability->GetSupportedSizes(0); + for (auto &pic : sizeList) { + if (pic.width == CAMERA_PIC_WIDTH && pic.height == CAMERA_PIC_HEIGHT) { + camId = cam; + break; + } + } + if (!camId.empty()) { + break; + } + } + if (camId.empty()) { + printf("No available camera.(1080p wanted)\n"); + return IC_RETCODE_FAILURE; + } + + EventHandler eventHdlr; + SampleCameraStateMng camStateMng(eventHdlr); + camKit->CreateCamera(camId, camStateMng, eventHdlr); + std::unique_lock cameraCreateLock(g_mutex); + g_cv.wait(cameraCreateLock); + cameraCreateLock.unlock(); + camStateMng.Capture(); + std::unique_lock captureLock(g_mutex); + g_cv.wait(captureLock); + captureLock.unlock(); + printf("Camera sample end.\n"); + return IC_RETCODE_SUCCESS; +} + +static int ImageClassificationExecute(const IcInput &picData) +{ + if (picData.data == nullptr || picData.size == 0) { + printf("ImageClassificationExecute: picData is empty.\n"); + return IC_RETCODE_FAILURE; + } + shared_ptr detector = make_shared(); + shared_ptr callback = make_shared(); + if (detector == nullptr || callback == nullptr) { + return IC_RETCODE_FAILURE; + } + detector->SetCallback(callback); + int retcode = detector->Create(); + if (retcode != IC_RETCODE_SUCCESS) { + printf("ImageClassificationExecute: IcSdk Create failed.\n"); + return IC_RETCODE_FAILURE; + } + retcode = detector->SyncExecute(picData); + if (retcode != IC_RETCODE_SUCCESS) { + printf("ImageClassificationExecute: IcSdk SyncExecute failed.\n"); + } + (void)detector->Destroy(); + return retcode; +} + +static int GetPictureData(const string &sourcePath, Array &picData) +{ + int srcWidth = 0; + int srcHeight = 0; + uint8_t *rawData = ReadJpegFile(sourcePath, srcWidth, srcHeight); + if (rawData == nullptr) { + printf("ReadJpegFile failed.\n"); + return IC_RETCODE_FAILURE; + } + uint8_t *srcData = Resize(WIDTH_DEST, HEIGHT_DEST, rawData, srcWidth, srcHeight); + if (srcData == nullptr) { + printf("Resize failed.\n"); + DELETE_PIC_BUFFER(rawData); + return IC_RETCODE_FAILURE; + } + int srcDataSize = WIDTH_DEST * HEIGHT_DEST * NUM_CHANNELS; + uint8_t *input = ConvertToCaffeInput(srcData, srcDataSize); + if (input == nullptr) { + printf("Convert to caffe input failed.\n"); + DELETE_PIC_BUFFER(rawData); + DELETE_PIC_BUFFER(srcData); + return IC_RETCODE_FAILURE; + } + picData.data = input; + picData.size = srcDataSize; + DELETE_PIC_BUFFER(rawData); + DELETE_PIC_BUFFER(srcData); + return IC_RETCODE_SUCCESS; +} + +static int GetPicFromCamera(IcInput &picData) +{ + int retcode = CameraCapture(); + if (retcode != IC_RETCODE_SUCCESS) { + printf("Capture picture failed.\n"); + return IC_RETCODE_FAILURE; + } + retcode = GetPictureData(CAMERA_SAVE_PATH, picData); + if (retcode != IC_RETCODE_SUCCESS) { + printf("GetPictureData failed.\n"); + return IC_RETCODE_FAILURE; + } + return IC_RETCODE_SUCCESS; +} + +static int GetPicFromLocal(IcInput &picData) +{ + int retcode = GetPictureData(JPEG_SRC_PATH, picData); + if (retcode != IC_RETCODE_SUCCESS) { + printf("GetPictureData failed.\n"); + return IC_RETCODE_FAILURE; + } + return IC_RETCODE_SUCCESS; +} + +static void SampleHelp() +{ + printf("****************************************\n"); + printf("Select the image source.\n"); + printf("1: Camera capture\n"); + printf("2: Local\n"); + printf("****************************************\n"); +} + +int main() +{ + IcInput picData = { + .data = nullptr, + .size = 0 + }; + int retcode = IC_RETCODE_SUCCESS; + char input = ' '; + SampleHelp(); + cin >> input; + switch (input) { + case '1': + retcode = GetPicFromCamera(picData); + break; + case '2': + retcode = GetPicFromLocal(picData); + break; + default: + SampleHelp(); + break; + } + if (retcode != IC_RETCODE_SUCCESS) { + printf("GetPicture failed.\n"); + return -1; + } + if (retcode != IC_RETCODE_SUCCESS) { + printf("WriteBgrFile failed.\n"); + DELETE_PIC_BUFFER(picData.data); + return -1; + } + retcode = ImageClassificationExecute(picData); + if (retcode != IC_RETCODE_SUCCESS) { + printf("ImageClassification failed.\n"); + } else { + printf("ImageClassification successed.\n"); + } + DELETE_PIC_BUFFER(picData.data); + return 0; +} \ No newline at end of file diff --git a/ai/cv/image_classification/picture_utils.cpp b/ai/cv/image_classification/picture_utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c6b222c0c83a47ee231388c97f5056662f8ef748 --- /dev/null +++ b/ai/cv/image_classification/picture_utils.cpp @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "picture_utils.h" + +#include +#include + +#include "ic_retcode.h" +#include "resize_computer.h" +#include "securec.h" + +using namespace std; + +namespace IC { +int WriteJpegFile(const string &filename, int quality, + uint8_t *srcBuffer, int srcWidth, int srcHeight) +{ + struct jpeg_compress_struct cinfo; + struct jpeg_error_mgr jerr; + JSAMPROW rowPointer[1]; + int rowStride = 0; + cinfo.err = jpeg_std_error(&jerr); + jpeg_create_compress(&cinfo); + if (srcBuffer == nullptr) { + printf("WriteJpegFile: srcBuffer is nullptr\n"); + return IC_RETCODE_FAILURE; + } + FILE *outfile; + if ((outfile = fopen(filename.c_str(), "wb")) == nullptr) { + printf("WriteJpegFile: can't open %s\n", filename.c_str()); + return IC_RETCODE_FAILURE; + } + jpeg_stdio_dest(&cinfo, outfile); + cinfo.image_width = srcWidth; + cinfo.image_height = srcHeight; + cinfo.input_components = NUM_CHANNELS; + cinfo.in_color_space = JCS_RGB; + jpeg_set_defaults(&cinfo); + jpeg_set_quality(&cinfo, quality, TRUE); + jpeg_start_compress(&cinfo, TRUE); + rowStride = srcWidth * NUM_CHANNELS; + + while (cinfo.next_scanline < cinfo.image_height) { + rowPointer[0] = &srcBuffer[cinfo.next_scanline * rowStride]; + (void)jpeg_write_scanlines(&cinfo, rowPointer, 1); + } + jpeg_finish_compress(&cinfo); + + fclose(outfile); + jpeg_destroy_compress(&cinfo); + return IC_RETCODE_SUCCESS; +} + +int WriteBgrFile(const string &filename, uint8_t *dataBuffer, int bufferSize) +{ + if (dataBuffer == nullptr || bufferSize <= 0) { + printf("WriteBgrFile: dataBuffer is nullptr.\n"); + return IC_RETCODE_FAILURE; + } + ofstream outfile(filename.c_str(), ofstream::out | ofstream::trunc); + if (!outfile.is_open()) { + printf("WriteBgrFile: Error writing file from BGR dataBuffer\n"); + return IC_RETCODE_FAILURE; + } + outfile.write((const char*)dataBuffer, bufferSize); + outfile.close(); + return IC_RETCODE_SUCCESS; +} + +uint8_t *ConvertToCaffeInput(uint8_t *dataBuffer, int maxSize) +{ + if (dataBuffer == nullptr) { + return nullptr; + } + if (maxSize % NUM_CHANNELS != 0) { + return nullptr; + } + uint8_t *input = new (std::nothrow) uint8_t[maxSize]; + if (input == nullptr) { + return nullptr; + } + int numPreChannel = maxSize / NUM_CHANNELS; + for (int i = 0; i < maxSize; i++) { + input[BGR_RED * numPreChannel + i / NUM_CHANNELS] = dataBuffer[i + RGB_RED]; + input[BGR_BLUE * numPreChannel + i / NUM_CHANNELS] = dataBuffer[i + RGB_BLUE]; + input[BGR_GREEN * numPreChannel + i / NUM_CHANNELS] = dataBuffer[i + RGB_GREEN]; + } + return input; +} + +uint8_t *ReadJpegFile(const string &filename, int &srcWidth, int &srcHeight) +{ + struct jpeg_decompress_struct cinfo; + struct MyErrorMgr jerr; + FILE *infile; + if ((infile = fopen(filename.c_str(), "rb")) == nullptr) { + printf("ReadJpegFile: can't open %s\n", filename.c_str()); + return nullptr; + } + + cinfo.err = jpeg_std_error(&jerr.pub); + jpeg_create_decompress(&cinfo); + jpeg_stdio_src(&cinfo, infile); + (void)jpeg_read_header(&cinfo, TRUE); + (void)jpeg_start_decompress(&cinfo); + srcHeight = cinfo.output_height; + srcWidth = cinfo.output_width; + int dataSize = srcHeight * srcWidth * cinfo.output_components; + uint8_t *buffer = new (std::nothrow) uint8_t[dataSize]; + if (buffer == nullptr) { + printf("ReadJpegFile: error to alloc buffer.\n"); + (void)jpeg_finish_decompress(&cinfo); + jpeg_destroy_decompress(&cinfo); + fclose(infile); + return nullptr; + } + uint8_t *rowptr = nullptr; + while (cinfo.output_scanline < srcHeight) { + rowptr = buffer + cinfo.output_scanline * srcWidth * cinfo.output_components; + (void)jpeg_read_scanlines(&cinfo, &rowptr, 1); + } + + (void)jpeg_finish_decompress(&cinfo); + jpeg_destroy_decompress(&cinfo); + fclose(infile); + return buffer; +} + +uint8_t *Resize( + const int widthDest, const int heightDest, uint8_t *src, int widthSrc, int heightSrc) +{ + if (src == nullptr) { + printf("Resize: src is nullptr.\n"); + return nullptr; + } + if (widthDest <= 0 || heightDest <= 0 || widthSrc <= 0 || heightSrc <= 0) { + printf("Resize: dimension below zero.\n"); + return nullptr; + } + int bufferSize = widthDest * heightDest * NUM_CHANNELS; + uint8_t *pDest = new (std::nothrow) uint8_t[bufferSize]; + PicInfo picInfo = { + .widthSrc = widthSrc, + .heightSrc = heightSrc, + .widthDest = widthDest, + .heightDest = heightDest + }; + ResizeComputer resizer(picInfo); + if (pDest == nullptr) { + printf("Resize: pDest alloc failed.\n"); + return nullptr; + } + if (widthDest == widthSrc && heightDest == heightSrc) { + if (memcpy_s(pDest, bufferSize, src, bufferSize) != EOK) { + printf("Resize: memcpy_s failed.\n"); + delete[] pDest; + return nullptr; + } else { + return pDest; + } + } + resizer.Compute(pDest, src, bufferSize, widthSrc * heightSrc * NUM_CHANNELS); + return pDest; +} +} // namespace IC \ No newline at end of file diff --git a/ai/cv/image_classification/picture_utils.h b/ai/cv/image_classification/picture_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..dad7381de71d09168831cbb4887f3d1d1d899f56 --- /dev/null +++ b/ai/cv/image_classification/picture_utils.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PICTURE_UTILS_H +#define PICTURE_UTILS_H + +#include +#include +#include +#include + +#include "jpeglib.h" + +namespace IC { +struct MyErrorMgr { + struct jpeg_error_mgr pub; + jmp_buf setjmp_buffer; +}; + +using MyErrorPtr = MyErrorMgr *; + +// The index order of BGR format +enum BgrIndex { + BGR_BLUE = 0, + BGR_GREEN, + BGR_RED, +}; + +// The index order of RGB format +enum RgbIndex { + RGB_RED = 0, + RGB_GREEN, + RGB_BLUE, +}; + +// Information about picture +struct PicInfo { + int widthSrc; + int heightSrc; + int widthDest; + int heightDest; +}; + +// Change this to your own settings +const std::string JPEG_SRC_PATH = "/storage/data/image_classification_demo.jpg"; +const int WIDTH_DEST = 224; +const int HEIGHT_DEST = 224; +const int NUM_CHANNELS = 3; + +int WriteJpegFile(const std::string &filename, int quality, uint8_t *srcBuffer, int srcWidth, int srcHeight); +int WriteBgrFile(const std::string &filename, uint8_t *dataBuffer, int bufferSize); +uint8_t *ConvertToCaffeInput(uint8_t *dataBuffer, int maxSize); +uint8_t *ReadJpegFile(const std::string &filename, int &srcWidth, int &srcHeight); +uint8_t *Resize(const int widthDest, const int heightDest, uint8_t *src, int widthSrc, int heightSrc); +} // namespace IC +#endif // PICTURE_UTILS_H diff --git a/ai/cv/image_classification/resize_computer.cpp b/ai/cv/image_classification/resize_computer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c119b5ca2a0e849fe27e10fcb8094e2cc38d57cb --- /dev/null +++ b/ai/cv/image_classification/resize_computer.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ic_retcode.h" +#include "resize_computer.h" + +namespace IC { +const int RESIZE_CONSTANT = 2; + +ResizeComputer::ResizeComputer(const PicInfo &pInfo) +{ + // Initialize parameter for resize computer + sw_ = pInfo.widthSrc - 1; + sh_ = pInfo.heightSrc - 1; + dw_ = pInfo.widthDest - 1; + dh_ = pInfo.heightDest - 1; + B_ = 0; + N_ = 0; + x_ = 0; + y_ = 0; + pLinePrev_ = nullptr; + pLineNext_ = nullptr; + pA_ = nullptr; + pB_ = nullptr; + pC_ = nullptr; + pD_ = nullptr; +} + +int ResizeComputer::Compute(uint8_t *pDest, uint8_t *src, int pDestSize, int srcSize) +{ + if (pDest == nullptr || src == nullptr || pDestSize <= 0 || srcSize <= 0) { + printf("ResizeComputer::Compute input is nullptr.\n"); + return IC_RETCODE_FAILURE; + } + uint8_t *tmp = nullptr; + + // This is linear stretch for picture resize + for (int i = 0; i <= dh_; ++i) { + tmp = pDest + i * (dw_ + 1) * NUM_CHANNELS; + y_ = i * sh_ / dh_; + N_ = dh_ - i * sh_ % dh_; + pLinePrev_ = src + y_ * (sw_ + 1) * NUM_CHANNELS; + y_++; + pLineNext_ = (N_ == dh_) ? pLinePrev_ : (src + y_ * (sw_ + 1) * NUM_CHANNELS); + for (int j = 0; j <= dw_; ++j) { + x_ = j * sw_ / dw_ * NUM_CHANNELS; + B_ = dw_ - j * sw_ % dw_; + pA_ = pLinePrev_ + x_; + pB_ = pA_ + NUM_CHANNELS; + pC_ = pLineNext_ + x_; + pD_ = pC_ + NUM_CHANNELS; + if (B_ == dw_) { + pB_ = pA_; + pD_ = pC_; + } + for (int k = 0; k < NUM_CHANNELS; ++k, ++tmp, ++pA_, ++pB_, ++pC_, ++pD_) { + *tmp = static_cast( + (B_ * N_ * (*pA_ - *pB_ - *pC_ + *pD_) + dw_ * N_ * (*pB_) + + dh_ * B_ * (*pC_) + (dw_ * dh_ - dh_ * B_ - dw_ * N_) * (*pD_) + + dw_ * dh_ / RESIZE_CONSTANT) / (dw_ * dh_)); + } + } + } + return IC_RETCODE_SUCCESS; +} +} // namespace IC \ No newline at end of file diff --git a/ai/cv/image_classification/resize_computer.h b/ai/cv/image_classification/resize_computer.h new file mode 100644 index 0000000000000000000000000000000000000000..8e76bc6c01cc2362e1e4db6465a25054435d9f9c --- /dev/null +++ b/ai/cv/image_classification/resize_computer.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RESIZE_COMPUTER_H +#define RESIZE_COMPUTER_H + +#include "picture_utils.h" + +namespace IC { +class ResizeComputer { +public: + explicit ResizeComputer(const PicInfo &pInfo); + ~ResizeComputer() = default; + int Compute(uint8_t *pDest, uint8_t *src, int pDestSize, int srcSize); + +private: + int sw_; + int sh_; + int dw_; + int dh_; + int B_; + int N_; + int x_; + int y_; + uint8_t *pLinePrev_; + uint8_t *pLineNext_; + uint8_t *pA_; + uint8_t *pB_; + uint8_t *pC_; + uint8_t *pD_; +}; +} // namespace IC +#endif // RESIZE_COMPUTER_H \ No newline at end of file diff --git a/ai/cv/image_classification/resources/image_classification_demo.jpg b/ai/cv/image_classification/resources/image_classification_demo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b65cf13ce88cdefc0f6a56c660bad8cec3791135 Binary files /dev/null and b/ai/cv/image_classification/resources/image_classification_demo.jpg differ diff --git a/ai/cv/image_classification/resources/synset_words.txt b/ai/cv/image_classification/resources/synset_words.txt new file mode 100644 index 0000000000000000000000000000000000000000..4ae0b014ed7c42005eaea024776db53586e6ca7c --- /dev/null +++ b/ai/cv/image_classification/resources/synset_words.txt @@ -0,0 +1,1000 @@ +tench, Tinca tinca +goldfish, Carassius auratus +great white shark, white shark, man-eater, man-eating shark, Carcharodon +tiger shark, Galeocerdo cuvieri +hammerhead, hammerhead shark +electric ray, crampfish, numbfish, torpedo +stingray +cock +hen +ostrich, Struthio camelus +brambling, Fringilla montifringilla +goldfinch, Carduelis carduelis +house finch, linnet, Carpodacus mexicanus +junco, snowbird +indigo bunting, indigo finch, indigo bird, Passerina cyanea +robin, American robin, Turdus migratorius +bulbul +jay +magpie +chickadee +water ouzel, dipper +kite +bald eagle, American eagle, Haliaeetus leucocephalus +vulture +great grey owl, great gray owl, Strix nebulosa +European fire salamander, Salamandra salamandra +common newt, Triturus vulgaris +eft +spotted salamander, Ambystoma maculatum +axolotl, mud puppy, Ambystoma mexicanum +bullfrog, Rana catesbeiana +tree frog, tree-frog +tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +loggerhead, loggerhead turtle, Caretta caretta +leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +mud turtle +terrapin +box turtle, box tortoise +banded gecko +common iguana, iguana, Iguana iguana +American chameleon, anole, Anolis carolinensis +whiptail, whiptail lizard +agama +frilled lizard, Chlamydosaurus kingi +alligator lizard +Gila monster, Heloderma suspectum +green lizard, Lacerta viridis +African chameleon, Chamaeleo chamaeleon +Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +African crocodile, Nile crocodile, Crocodylus niloticus +American alligator, Alligator mississipiensis +triceratops +thunder snake, worm snake, Carphophis amoenus +ringneck snake, ring-necked snake, ring snake +hognose snake, puff adder, sand viper +green snake, grass snake +king snake, kingsnake +garter snake, grass snake +water snake +vine snake +night snake, Hypsiglena torquata +boa constrictor, Constrictor constrictor +rock python, rock snake, Python sebae +Indian cobra, Naja naja +green mamba +sea snake +horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +diamondback, diamondback rattlesnake, Crotalus adamanteus +sidewinder, horned rattlesnake, Crotalus cerastes +trilobite +harvestman, daddy longlegs, Phalangium opilio +scorpion +black and gold garden spider, Argiope aurantia +barn spider, Araneus cavaticus +garden spider, Aranea diademata +black widow, Latrodectus mactans +tarantula +wolf spider, hunting spider +tick +centipede +black grouse +ptarmigan +ruffed grouse, partridge, Bonasa umbellus +prairie chicken, prairie grouse, prairie fowl +peacock +quail +partridge +African grey, African gray, Psittacus erithacus +macaw +sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser, Mergus serrator +goose +black swan, Cygnus atratus +tusker +echidna, spiny anteater, anteater +platypus, duckbill, duckbilled platypus, duck-billed platypus, nchus anatinus +wallaby, brush kangaroo +koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +wombat +jellyfish +sea anemone, anemone +brain coral +flatworm, platyhelminth +nematode, nematode worm, roundworm +conch +snail +slug +sea slug, nudibranch +chiton, coat-of-mail shell, sea cradle, polyplacophore +chambered nautilus, pearly nautilus, nautilus +Dungeness crab, Cancer magister +rock crab, Cancer irroratus +fiddler crab +king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes ca +American lobster, Northern lobster, Maine lobster, Homarus americanus +spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +crayfish, crawfish, crawdad, crawdaddy +hermit crab +isopod +white stork, Ciconia ciconia +black stork, Ciconia nigra +spoonbill +flamingo +little blue heron, Egretta caerulea +American egret, great white heron, Egretta albus +bittern +crane +limpkin, Aramus pictus +European gallinule, Porphyrio porphyrio +American coot, marsh hen, mud hen, water hen, Fulica americana +bustard +ruddy turnstone, Arenaria interpres +red-backed sandpiper, dunlin, Erolia alpina +redshank, Tringa totanus +dowitcher +oystercatcher, oyster catcher +pelican +king penguin, Aptenodytes patagonica +albatross, mollymawk +grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius +killer whale, killer, orca, grampus, sea wolf, Orcinus orca +dugong, Dugong dugon +sea lion +Chihuahua +Japanese spaniel +Maltese dog, Maltese terrier, Maltese +Pekinese, Pekingese, Peke +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound, Afghan +basset, basset hound +beagle +bloodhound, sleuthhound +bluetick +black-and-tan coonhound +Walker hound, Walker foxhound +English foxhound +redbone +borzoi, Russian wolfhound +Irish wolfhound +Italian greyhound +whippet +Ibizan hound, Ibizan Podenco +Norwegian elkhound, elkhound +otterhound, otter hound +Saluki, gazelle hound +Scottish deerhound, deerhound +Weimaraner +Staffordshire bullterrier, Staffordshire bull terrier +American Staffordshire terrier, Staffordshire terrier, American pit bull it bull terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier, Sealyham +Airedale, Airedale terrier +cairn, cairn terrier +Australian terrier +Dandie Dinmont, Dandie Dinmont terrier +Boston bull, Boston terrier +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier, Scottish terrier, Scottie +Tibetan terrier, chrysanthemum dog +silky terrier, Sydney silky +soft-coated wheaten terrier +West Highland white terrier +Lhasa, Lhasa apso +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla, Hungarian pointer +English setter +Irish setter, red setter +Gordon setter +Brittany spaniel +clumber, clumber spaniel +English springer, English springer spaniel +Welsh springer spaniel +cocker spaniel, English cocker spaniel, cocker +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog, bobtail +Shetland sheepdog, Shetland sheep dog, Shetland +collie +Border collie +Bouvier des Flandres, Bouviers des Flandres +Rottweiler +German shepherd, German shepherd dog, German police dog, alsatian +Doberman, Doberman pinscher +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard, St Bernard +Eskimo dog, husky +malamute, malemute, Alaskan malamute +Siberian husky +dalmatian, coach dog, carriage dog +affenpinscher, monkey pinscher, monkey dog +basenji +pug, pug-dog +Leonberg +Newfoundland, Newfoundland dog +Great Pyrenees +Samoyed, Samoyede +Pomeranian +chow, chow chow +keeshond +Brabancon griffon +Pembroke, Pembroke Welsh corgi +Cardigan, Cardigan Welsh corgi +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf, grey wolf, gray wolf, Canis lupus +white wolf, Arctic wolf, Canis lupus tundrarum +red wolf, maned wolf, Canis rufus, Canis niger +coyote, prairie wolf, brush wolf, Canis latrans +dingo, warrigal, warragal, Canis dingo +dhole, Cuon alpinus +African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +hyena, hyaena +red fox, Vulpes vulpes +kit fox, Vulpes macrotis +Arctic fox, white fox, Alopex lagopus +grey fox, gray fox, Urocyon cinereoargenteus +tabby, tabby cat +tiger cat +Persian cat +Siamese cat, Siamese +Egyptian cat +cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +lynx, catamount +leopard, Panthera pardus +snow leopard, ounce, Panthera uncia +jaguar, panther, Panthera onca, Felis onca +lion, king of beasts, Panthera leo +tiger, Panthera tigris +cheetah, chetah, Acinonyx jubatus +brown bear, bruin, Ursus arctos +American black bear, black bear, Ursus americanus, Euarctos americanus +ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +sloth bear, Melursus ursinus, Ursus ursinus +mongoose +meerkat, mierkat +tiger beetle +ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +ground beetle, carabid beetle +long-horned beetle, longicorn, longicorn beetle +leaf beetle, chrysomelid +dung beetle +rhinoceros beetle +weevil +fly +bee +ant, emmet, pismire +grasshopper, hopper +cricket +walking stick, walkingstick, stick insect +cockroach, roach +mantis, mantid +cicada, cicala +leafhopper +lacewing, lacewing fly +dragonfly, darning needle, devils darning needle, sewing needle, snake ake doctor, mosquito hawk, skeeter hawk +damselfly +admiral +ringlet, ringlet butterfly +monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +cabbage butterfly +sulphur butterfly, sulfur butterfly +lycaenid, lycaenid butterfly +starfish, sea star +sea urchin +sea cucumber, holothurian +wood rabbit, cottontail, cottontail rabbit +hare +Angora, Angora rabbit +hamster +porcupine, hedgehog +fox squirrel, eastern fox squirrel, Sciurus niger +marmot +beaver +guinea pig, Cavia cobaya +sorrel +zebra +hog, pig, grunter, squealer, Sus scrofa +wild boar, boar, Sus scrofa +warthog +hippopotamus, hippo, river horse, Hippopotamus amphibius +ox +water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +bison +ram, tup +bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain s canadensis +ibex, Capra ibex +hartebeest +impala, Aepyceros melampus +gazelle +Arabian camel, dromedary, Camelus dromedarius +llama +weasel +mink +polecat, fitch, foulmart, foumart, Mustela putorius +black-footed ferret, ferret, Mustela nigripes +otter +skunk, polecat, wood pussy +badger +armadillo +three-toed sloth, ai, Bradypus tridactylus +orangutan, orang, orangutang, Pongo pygmaeus +gorilla, Gorilla gorilla +chimpanzee, chimp, Pan troglodytes +gibbon, Hylobates lar +siamang, Hylobates syndactylus, Symphalangus syndactylus +guenon, guenon monkey +patas, hussar monkey, Erythrocebus patas +baboon +macaque +langur +colobus, colobus monkey +proboscis monkey, Nasalis larvatus +marmoset +capuchin, ringtail, Cebus capucinus +howler monkey, howler +titi, titi monkey +spider monkey, Ateles geoffroyi +squirrel monkey, Saimiri sciureus +Madagascar cat, ring-tailed lemur, Lemur catta +indri, indris, Indri indri, Indri brevicaudatus +Indian elephant, Elephas maximus +African elephant, Loxodonta africana +lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +barracouta, snoek +eel +coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +rock beauty, Holocanthus tricolor +anemone fish +sturgeon +gar, garfish, garpike, billfish, Lepisosteus osseus +lionfish +puffer, pufferfish, blowfish, globefish +abacus +abaya +academic gown, academic robe, judges robe +accordion, piano accordion, squeeze box +acoustic guitar +aircraft carrier, carrier, flattop, attack aircraft carrier +airliner +airship, dirigible +altar +ambulance +amphibian, amphibious vehicle +analog clock +apiary, bee house +apron +ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, el, trash bin +assault rifle, assault gun +backpack, back pack, knapsack, packsack, rucksack, haversack +bakery, bakeshop, bakehouse +balance beam, beam +balloon +ballpoint, ballpoint pen, ballpen, Biro +Band Aid +banjo +bannister, banister, balustrade, balusters, handrail +barbell +barber chair +barbershop +barn +barometer +barrel, cask +barrow, garden cart, lawn cart, wheelbarrow +baseball +basketball +bassinet +bassoon +bathing cap, swimming cap +bath towel +bathtub, bathing tub, bath, tub +beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, +beacon, lighthouse, beacon light, pharos +beaker +bearskin, busby, shako +beer bottle +beer glass +bell cote, bell cot +bib +bicycle-built-for-two, tandem bicycle, tandem +bikini, two-piece +binder, ring-binder +binoculars, field glasses, opera glasses +birdhouse +boathouse +bobsled, bobsleigh, bob +bolo tie, bolo, bola tie, bola +bonnet, poke bonnet +bookcase +bookshop, bookstore, bookstall +bottlecap +bow +bow tie, bow-tie, bowtie +brass, memorial tablet, plaque +brassiere, bra, bandeau +breakwater, groin, groyne, mole, bulwark, seawall, jetty +breastplate, aegis, egis +broom +bucket, pail +buckle +bulletproof vest +bullet train, bullet +butcher shop, meat market +cab, hack, taxi, taxicab +caldron, cauldron +candle, taper, wax light +cannon +canoe +can opener, tin opener +cardigan +car mirror +carousel, carrousel, merry-go-round, roundabout, whirligig +carpenters kit, tool kit +carton +car wheel +cash machine, cash dispenser, automated teller machine, automatic teller utomated teller, automatic teller, ATM +cassette +cassette player +castle +catamaran +CD player +cello, violoncello +cellular telephone, cellular phone, cellphone, cell, mobile phone +chain +chainlink fence +chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring +chain saw, chainsaw +chest +chiffonier, commode +chime, bell, gong +china cabinet, china closet +Christmas stocking +church, church building +cinema, movie theater, movie theatre, movie house, picture palace +cleaver, meat cleaver, chopper +cliff dwelling +cloak +clog, geta, patten, sabot +cocktail shaker +coffee mug +coffeepot +coil, spiral, volute, whorl, helix +combination lock +computer keyboard, keypad +confectionery, confectionary, candy store +container ship, containership, container vessel +convertible +corkscrew, bottle screw +cornet, horn, trumpet, trump +cowboy boot +cowboy hat, ten-gallon hat +cradle +crane +crash helmet +crate +crib, cot +Crock Pot +croquet ball +crutch +cuirass +dam, dike, dyke +desk +desktop computer +dial telephone, dial phone +diaper, nappy, napkin +digital clock +digital watch +dining table, board +dishrag, dishcloth +dishwasher, dish washer, dishwashing machine +disk brake, disc brake +dock, dockage, docking facility +dogsled, dog sled, dog sleigh +dome +doormat, welcome mat +drilling platform, offshore rig +drum, membranophone, tympan +drumstick +dumbbell +Dutch oven +electric fan, blower +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa, boa +file, file cabinet, filing cabinet +fireboat +fire engine, fire truck +fire screen, fireguard +flagpole, flagstaff +flute, transverse flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn, horn +frying pan, frypan, skillet +fur coat +garbage truck, dustcart +gasmask, respirator, gas helmet +gas pump, gasoline pump, petrol pump, island dispenser +goblet +go-kart +golf ball +golfcart, golf cart +gondola +gong, tam-tam +gown +grand piano, grand +greenhouse, nursery, glasshouse +grille, radiator grille +grocery store, grocery, food market, market +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower, blow dryer, blow drier, hair dryer, hair drier +hand-held computer, hand-held microcomputer +handkerchief, hankie, hanky, hankey +hard disc, hard disk, fixed disk +harmonica, mouth organ, harp, mouth harp +harp +harvester, reaper +hatchet +holster +home theater, home theatre +honeycomb +hook, claw +hoopskirt, crinoline +horizontal bar, high bar +horse cart, horse-cart +hourglass +iPod +iron, smoothing iron +jack-o-lantern +jean, blue jean, denim +jeep, landrover +jersey, T-shirt, tee shirt +jigsaw puzzle +jinrikisha, ricksha, rickshaw +joystick +kimono +knee pad +knot +lab coat, laboratory coat +ladle +lampshade, lamp shade +laptop, laptop computer +lawn mower, mower +lens cap, lens cover +letter opener, paper knife, paperknife +library +lifeboat +lighter, light, igniter, ignitor +limousine, limo +liner, ocean liner +lipstick, lip rouge +Loafer +lotion +loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +loupe, jewelers loupe +lumbermill, sawmill +magnetic compass +mailbag, postbag +mailbox, letter box +maillot +maillot, tank suit +manhole cover +maraca +marimba, xylophone +mask +matchstick +maypole +maze, labyrinth +measuring cup +medicine chest, medicine cabinet +megalith, megalithic structure +microphone, mike +microwave, microwave oven +military uniform +milk can +minibus +miniskirt, mini +minivan +missile +mitten +mixing bowl +mobile home, manufactured home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter, scooter +mountain bike, all-terrain bike, off-roader +mountain tent +mouse, computer mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook, notebook computer +obelisk +oboe, hautboy, hautbois +ocarina, sweet potato +odometer, hodometer, mileometer, milometer +oil filter +organ, pipe organ +oscilloscope, scope, cathode-ray oscilloscope, CRO +overskirt +oxcart +oxygen mask +packet +paddle, boat paddle +paddlewheel, paddle wheel +padlock +paintbrush +pajama, pyjama, pjs, jammies +palace +panpipe, pandean pipe, syrinx +paper towel +parachute, chute +parallel bars, bars +park bench +parking meter +passenger car, coach, carriage +patio, terrace +pay-phone, pay-station +pedestal, plinth, footstall +pencil box, pencil case +pencil sharpener +perfume, essence +Petri dish +photocopier +pick, plectrum, plectron +pickelhaube +picket fence, paling +pickup, pickup truck +pier +piggy bank, penny bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate, pirate ship +pitcher, ewer +plane, carpenters plane, woodworking plane +planetarium +plastic bag +plate rack +plow, plough +plunger, plumbers helper +Polaroid camera, Polaroid Land camera +pole +police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +poncho +pool table, billiard table, snooker table +pop bottle, soda bottle +pot, flowerpot +potters wheel +power drill +prayer rug, prayer mat +printer +prison, prison house +projectile, missile +projector +puck, hockey puck +punching bag, punch bag, punching ball, punchball +purse +quill, quill pen +quilt, comforter, comfort, puff +racer, race car, racing car +racket, racquet +radiator +radio, wireless +radio telescope, radio reflector +rain barrel +recreational vehicle, RV, R.V. +reel +reflex camera +refrigerator, icebox +remote control, remote +restaurant, eating house, eating place, eatery +revolver, six-gun, six-shooter +rifle +rocking chair, rocker +rotisserie +rubber eraser, rubber, pencil eraser +rugby ball +rule, ruler +running shoe +safe +safety pin +saltshaker, salt shaker +sandal +sarong +sax, saxophone +scabbard +scale, weighing machine +school bus +schooner +scoreboard +screen, CRT screen +screw +screwdriver +seat belt, seatbelt +sewing machine +shield, buckler +shoe shop, shoe-shop, shoe store +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule, slipstick +sliding door +slot, one-armed bandit +snorkel +snowmobile +snowplow, snowplough +soap dispenser +soccer ball +sock +solar dish, solar collector, solar furnace +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web, spiders web +spindle +sports car, sport car +spotlight, spot +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch, stop watch +stove +strainer +streetcar, tram, tramcar, trolley, trolley car +stretcher +studio couch, day bed +stupa, tope +submarine, pigboat, sub, U-boat +suit, suit of clothes +sundial +sunglass +sunglasses, dark glasses, shades +sunscreen, sunblock, sun blocker +suspension bridge +swab, swob, mop +sweatshirt +swimming trunks, bathing trunks +swing +switch, electric switch, electrical switch +syringe +table lamp +tank, army tank, armored combat vehicle, armoured combat vehicle +tape player +teapot +teddy, teddy bear +television, television system +tennis ball +thatch, thatched roof +theater curtain, theatre curtain +thimble +thresher, thrasher, threshing machine +throne +tile roof +toaster +tobacco shop, tobacconist shop, tobacconist +toilet seat +torch +totem pole +tow truck, tow car, wrecker +toyshop +tractor +trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +tray +trench coat +tricycle, trike, velocipede +trimaran +tripod +triumphal arch +trolleybus, trolley coach, trackless trolley +trombone +tub, vat +turnstile +typewriter keyboard +umbrella +unicycle, monocycle +upright, upright piano +vacuum, vacuum cleaner +vase +vault +velvet +vending machine +vestment +viaduct +violin, fiddle +volleyball +waffle iron +wall clock +wallet, billfold, notecase, pocketbook +wardrobe, closet, press +warplane, military plane +washbasin, handbasin, washbowl, lavabo, wash-hand basin +washer, automatic washer, washing machine +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool, woolen, woollen +worm fence, snake fence, snake-rail fence, Virginia fence +wreck +yawl +yurt +web site, website, internet site, site +comic book +crossword puzzle, crossword +street sign +traffic light, traffic signal, stoplight +book jacket, dust cover, dust jacket, dust wrapper +menu +plate +guacamole +consomme +hot pot, hotpot +trifle +ice cream, icecream +ice lolly, lolly, lollipop, popsicle +French loaf +bagel, beigel +pretzel +cheeseburger +hotdog, hot dog, red hot +mashed potato +head cabbage +broccoli +cauliflower +zucchini, courgette +spaghetti squash +acorn squash +butternut squash +cucumber, cuke +artichoke, globe artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple, ananas +banana +jackfruit, jak, jack +custard apple +pomegranate +hay +carbonara +chocolate sauce, chocolate syrup +dough +meat loaf, meatloaf +pizza, pizza pie +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff, drop, drop-off +coral reef +geyser +lakeside, lakeshore +promontory, headland, head, foreland +sandbar, sand bar +seashore, coast, seacoast, sea-coast +valley, vale +volcano +ballplayer, baseball player +groom, bridegroom +scuba diver +rapeseed +daisy +yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium m +corn +acorn +hip, rose hip, rosehip +buckeye, horse chestnut, conker +coral fungus +agaric +gyromitra +stinkhorn, carrion fungus +earthstar +hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +bolete +ear, spike, capitulum +toilet tissue, toilet paper, bathroom tissue \ No newline at end of file