[icinga-checkins] icinga.org: icinga2/master: CheckerComponent sometimes fails to schedule checks in time

git at icinga.org git at icinga.org
Thu Jul 28 12:52:34 CEST 2016


Module: icinga2
Branch: master
Commit: daf375b6d7577b1877aab4a3d719d3f2685e112e
URL:    https://git.icinga.org/?p=icinga2.git;a=commit;h=daf375b6d7577b1877aab4a3d719d3f2685e112e

Author: Gunnar Beutner <gunnar.beutner at netways.de>
Date:   Thu Jul 28 12:50:48 2016 +0200

CheckerComponent sometimes fails to schedule checks in time

fixes #12180

---

 lib/checker/checkercomponent.cpp |   39 +++++++++++++++++++++++++++++---------
 lib/checker/checkercomponent.hpp |   19 +++++++++++++++----
 2 files changed, 45 insertions(+), 13 deletions(-)

diff --git a/lib/checker/checkercomponent.cpp b/lib/checker/checkercomponent.cpp
index f335a91..eeb8249 100644
--- a/lib/checker/checkercomponent.cpp
+++ b/lib/checker/checkercomponent.cpp
@@ -117,12 +117,20 @@ void CheckerComponent::CheckThreadProc(void)
 			break;
 
 		CheckTimeView::iterator it = idx.begin();
-		Checkable::Ptr checkable = *it;
+		CheckableScheduleInfo csi = *it;
 
-		double wait = checkable->GetNextCheck() - Utility::GetTime();
+		double wait = csi.NextCheck - Utility::GetTime();
+
+		if (wait > 30) {
+			Log(LogWarning, "CheckerComponent")
+			    << "Wait: " << wait;
+		}
+
+		if (Checkable::GetPendingChecks() >= GetConcurrentChecks()) {
+			Log(LogWarning, "CheckerComponent", "Checkable::GetPendingChecks() >= GetConcurrentChecks()");
 
-		if (Checkable::GetPendingChecks() >= GetConcurrentChecks())
 			wait = 0.5;
+		}
 
 		if (wait > 0) {
 			/* Wait for the next check. */
@@ -131,6 +139,8 @@ void CheckerComponent::CheckThreadProc(void)
 			continue;
 		}
 
+		Checkable::Ptr checkable = csi.Object;
+
 		m_IdleCheckables.erase(checkable);
 
 		bool forced = checkable->GetForceNextCheck();
@@ -170,7 +180,7 @@ void CheckerComponent::CheckThreadProc(void)
 
 		/* reschedule the checkable if checks are disabled */
 		if (!check) {
-			m_IdleCheckables.insert(checkable);
+			m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
 			lock.unlock();
 
 			checkable->UpdateNextCheck();
@@ -180,7 +190,7 @@ void CheckerComponent::CheckThreadProc(void)
 			continue;
 		}
 
-		m_PendingCheckables.insert(checkable);
+		m_PendingCheckables.insert(GetCheckableScheduleInfo(checkable));
 
 		lock.unlock();
 
@@ -232,7 +242,7 @@ void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable)
 			m_PendingCheckables.erase(it);
 
 			if (checkable->IsActive())
-				m_IdleCheckables.insert(checkable);
+				m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
 
 			m_CV.notify_all();
 		}
@@ -250,7 +260,7 @@ void CheckerComponent::ResultTimerHandler(void)
 		boost::mutex::scoped_lock lock(m_Mutex);
 
 		msgbuf << "Pending checkables: " << m_PendingCheckables.size() << "; Idle checkables: " << m_IdleCheckables.size() << "; Checks/s: "
-		    << (CIB::GetActiveHostChecksStatistics(5) + CIB::GetActiveServiceChecksStatistics(5)) / 5.0;
+		    << (CIB::GetActiveHostChecksStatistics(60) + CIB::GetActiveServiceChecksStatistics(60)) / 60.0;
 	}
 
 	Log(LogNotice, "CheckerComponent", msgbuf.str());
@@ -273,7 +283,7 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
 			if (m_PendingCheckables.find(checkable) != m_PendingCheckables.end())
 				return;
 
-			m_IdleCheckables.insert(checkable);
+			m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
 		} else {
 			m_IdleCheckables.erase(checkable);
 			m_PendingCheckables.erase(checkable);
@@ -283,6 +293,14 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
 	}
 }
 
+CheckableScheduleInfo CheckerComponent::GetCheckableScheduleInfo(const Checkable::Ptr& checkable)
+{
+	CheckableScheduleInfo csi;
+	csi.Object = checkable;
+	csi.NextCheck = checkable->GetNextCheck();
+	return csi;
+}
+
 void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
 {
 	boost::mutex::scoped_lock lock(m_Mutex);
@@ -296,7 +314,10 @@ void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
 		return;
 
 	idx.erase(checkable);
-	idx.insert(checkable);
+
+	CheckableScheduleInfo csi = GetCheckableScheduleInfo(checkable);
+	idx.insert(csi);
+
 	m_CV.notify_all();
 }
 
diff --git a/lib/checker/checkercomponent.hpp b/lib/checker/checkercomponent.hpp
index 9896685..0e06d5e 100644
--- a/lib/checker/checkercomponent.hpp
+++ b/lib/checker/checkercomponent.hpp
@@ -38,6 +38,15 @@ namespace icinga
 /**
  * @ingroup checker
  */
+struct CheckableScheduleInfo
+{
+	Checkable::Ptr Object;
+	double NextCheck;
+};
+
+/**
+ * @ingroup checker
+ */
 struct CheckableNextCheckExtractor
 {
 	typedef double result_type;
@@ -45,9 +54,9 @@ struct CheckableNextCheckExtractor
 	/**
 	 * @threadsafety Always.
 	 */
-	double operator()(const Checkable::Ptr& checkable)
+	double operator()(const CheckableScheduleInfo& csi)
 	{
-		return checkable->GetNextCheck();
+		return csi.NextCheck;
 	}
 };
 
@@ -61,9 +70,9 @@ public:
 	DECLARE_OBJECTNAME(CheckerComponent);
 
 	typedef boost::multi_index_container<
-		Checkable::Ptr,
+		CheckableScheduleInfo,
 		boost::multi_index::indexed_by<
-			boost::multi_index::ordered_unique<boost::multi_index::identity<Checkable::Ptr> >,
+			boost::multi_index::ordered_unique<boost::multi_index::member<CheckableScheduleInfo, Checkable::Ptr, &CheckableScheduleInfo::Object> >,
 			boost::multi_index::ordered_non_unique<CheckableNextCheckExtractor>
 		>
 	> CheckableSet;
@@ -100,6 +109,8 @@ private:
 	void NextCheckChangedHandler(const Checkable::Ptr& checkable);
 
 	void RescheduleCheckTimer(void);
+
+	static CheckableScheduleInfo GetCheckableScheduleInfo(const Checkable::Ptr& checkable);
 };
 
 }



More information about the icinga-checkins mailing list